fuze-eletrik commited on
Commit
002abca
·
verified ·
1 Parent(s): f5d0555

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +22 -218
app.py CHANGED
@@ -19,17 +19,8 @@ PAGE = r"""
19
  #sam-browser-app .step{display:grid;place-items:center;flex:0 0 34px;height:34px;border-radius:11px;background:color-mix(in srgb,var(--sam-accent) 14%,transparent);color:var(--sam-accent2);font-weight:800}
20
  #sam-browser-app .muted{opacity:.68;font-size:13px;line-height:1.5}
21
  #sam-browser-app label{display:block;margin:0 0 7px;font-size:13px;font-weight:700}
22
- #sam-browser-app input[type=file],#sam-browser-app input[type=text],#sam-browser-app input[type=number],#sam-browser-app select,#sam-browser-app textarea{width:100%;min-width:0;padding:12px 13px;border-radius:10px;border:1px solid var(--border-color-primary);background:var(--input-background-fill);color:var(--body-text-color);font:inherit;font-size:14px}
23
  #sam-browser-app input[type=file]{padding:10px}
24
- #sam-browser-app textarea{width:100%;min-height:86px;resize:vertical;padding:11px 13px;border-radius:10px;border:1px solid var(--border-color-primary);background:var(--input-background-fill);color:var(--body-text-color);font:inherit;font-size:13px;line-height:1.5}
25
- #sam-browser-app .advanced{margin-top:18px;border:1px solid var(--border-color-primary);border-radius:13px;overflow:hidden}
26
- #sam-browser-app .advanced summary{cursor:pointer;padding:13px 15px;font-size:13px;font-weight:750;background:var(--background-fill-secondary)}
27
- #sam-browser-app .advanced-body{padding:15px;display:grid;gap:15px}
28
- #sam-browser-app .option-row{display:flex;gap:10px;align-items:flex-start}
29
- #sam-browser-app .option-row input{margin-top:3px;accent-color:var(--sam-accent)}
30
- #sam-browser-app .option-row label{margin:0;font-weight:750}
31
- #sam-browser-app .advanced-grid{display:grid;grid-template-columns:minmax(0,1fr) 120px;gap:12px;align-items:end}
32
- #sam-browser-app .threshold{width:100%;accent-color:var(--sam-accent)}
33
  #sam-browser-app .file-wrap{padding:16px;border:1px dashed var(--border-color-primary);border-radius:14px;background:var(--background-fill-secondary)}
34
  #sam-browser-app .controls{display:grid;grid-template-columns:minmax(0,2fr) minmax(150px,1fr) minmax(110px,.7fr);gap:14px;margin-top:18px}
35
  #sam-browser-app .field{min-width:0}
@@ -60,34 +51,28 @@ PAGE = r"""
60
  #sam-browser-app .empty-results{padding:24px 14px;border:1px dashed var(--border-color-primary);border-radius:14px;text-align:center;color:var(--body-text-color);opacity:.65;font-size:13px}
61
  #sam-browser-app .privacy{display:flex;gap:10px;align-items:flex-start;padding:12px 14px;border-radius:11px;background:color-mix(in srgb,#35b67a 8%,transparent);font-size:12px;line-height:1.5;margin-top:16px}
62
  #sam-browser-app .privacy svg{flex:0 0 17px;margin-top:1px}
63
- #sam-browser-app .footer-note{font-size:12px;opacity:.58;text-align:center;margin:18px 0 28px;line-height:1.5}
 
 
 
64
  @media(max-width:820px){#sam-browser-app .audio-grid{grid-template-columns:1fr}#sam-browser-app .controls{grid-template-columns:1fr 1fr}#sam-browser-app .controls .prompt-field{grid-column:1/-1}}
65
- @media(max-width:480px){#sam-browser-app .advanced-grid{grid-template-columns:1fr}#sam-browser-app .controls{grid-template-columns:1fr}#sam-browser-app .controls .prompt-field{grid-column:auto}#sam-browser-app .actions{align-items:stretch}#sam-browser-app .actions button{width:100%}#sam-browser-app .status{flex-basis:100%}}
66
  </style>
67
  <header class="hero">
68
  <div class="eyebrow">Audio separation studio</div>
69
  <h1>SAM Audio <span style="color:var(--sam-accent2)">Studio</span></h1>
70
- <p class="subhead">Separate speech, music, or other sounds from a recording with SAM Audio Small. Preview the original and both outputs with interactive waveforms, right in your browser.</p>
71
  <div class="pill"><span class="dot"></span> Local browser processing <span style="opacity:.45">·</span> ONNX Runtime</div>
72
  </header>
73
  <section class="panel">
74
  <div class="section-head"><div class="step">01</div><div><h2>Source & settings</h2><div class="muted">Choose a recording and describe what you want to isolate.</div></div></div>
75
- <div class="file-wrap"><label for="sam-audio-file">Audio file</label><input id="sam-audio-file" type="file" accept="audio/*,.wav,.mp3,.flac,.ogg,.m4a"><div class="muted" style="margin-top:8px">WAV, MP3, FLAC, OGG, or M4A · Maximum duration: 60 seconds</div></div>
76
  <div class="controls">
77
  <div class="field prompt-field"><label for="sam-prompt">What should be isolated?</label><input id="sam-prompt" type="text" value="a person speaking" placeholder="e.g. a person speaking"></div>
78
  <div class="field"><label for="sam-provider">Processing device</label><select id="sam-provider"><option value="auto" selected>Automatic</option><option value="webgpu">GPU · WebGPU</option><option value="wasm">CPU · WASM</option></select></div>
79
  <div class="field"><label for="sam-steps">ODE steps</label><input id="sam-steps" type="number" min="1" max="16" value="2"></div>
80
  </div>
81
  <div class="muted" style="margin-top:9px">Automatic mode uses CPU on phones/tablets and WebGPU on desktop when available. More ODE steps can improve quality but take longer.</div>
82
- <details class="advanced">
83
- <summary>Extra settings · Span prediction & anchors</summary>
84
- <div class="advanced-body">
85
- <div class="option-row"><input id="sam-predict-spans" type="checkbox"><div><label for="sam-predict-spans">Automatic Span Prediction</label><div class="muted">Detect when the described sound occurs and guide separation toward those time ranges. First use may download about 5.8 GB of additional PEAFrame weights plus tokenizer assets; browser/device memory limits may prevent loading them.</div></div></div>
86
- <div class="advanced-grid" id="sam-span-threshold-wrap" hidden><div><label for="sam-span-threshold">Detection threshold</label><input id="sam-span-threshold" class="threshold" type="range" min="0.05" max="0.95" step="0.05" value="0.30"></div><div id="sam-span-threshold-value" class="muted">0.30</div></div>
87
- <div class="option-row"><input id="sam-manual-anchors-enabled" type="checkbox"><div><label for="sam-manual-anchors-enabled">Manual Anchors</label><div class="muted">Define positive ranges to focus on (+) or negative ranges to ignore (−). Each line uses: sign start_seconds end_seconds.</div></div></div>
88
- <div id="sam-manual-anchors-wrap" hidden><label for="sam-manual-anchors">Anchor ranges</label><textarea id="sam-manual-anchors" spellcheck="false" placeholder="+ 4.5 7.0&#10;+ 12.0 15.5&#10;- 0.0 3.0"></textarea><div class="muted">Example: <code>+ 4.5 7.0</code> focuses on 4.5–7 seconds. Manual anchors take priority over automatic prediction.</div></div>
89
- </div>
90
- </details>
91
  <div class="actions"><button id="sam-run">▶ &nbsp; Separate audio</button><button id="sam-clear-cache" class="secondary" type="button">Clear model cache</button><div id="sam-status-box" class="status" data-state="idle"><span class="status-dot"></span><span id="sam-status" aria-live="polite">Ready when you are</span></div></div>
92
  <div class="privacy"><svg viewBox="0 0 24 24" width="17" height="17" fill="none" stroke="currentColor" stroke-width="2"><rect x="4" y="10" width="16" height="11" rx="2"/><path d="M8 10V7a4 4 0 0 1 8 0v3"/></svg><div><strong>Your audio stays on your device.</strong> Inference runs locally in this browser. The Space hosts the interface and model downloads; your selected audio is not uploaded for processing.</div></div>
93
  </section>
@@ -99,7 +84,8 @@ PAGE = r"""
99
  <article class="audio-card"><div class="audio-card-head"><span class="audio-title">Residual</span><span class="tag neutral">Everything else</span></div><div class="wave-wrap"><canvas id="sam-residual-wave" aria-label="Residual audio waveform"></canvas><div class="wave-empty" id="sam-residual-empty">Run separation to create the residual</div></div><div class="audio-meta"><span id="sam-residual-duration">Waiting for result</span><span>48 kHz · WAV</span></div><audio id="sam-residual-player" controls preload="metadata"></audio><a class="download" id="sam-residual-download" download="residual.wav" hidden>↓ Download residual.wav</a></article>
100
  </div>
101
  </section>
102
- <p class="footer-note">Experimental browser port · First use downloads model files and may take several minutes on CPU. Results should be compared with the Python reference before production use.</p>
 
103
  </div>
104
  """
105
 
@@ -119,9 +105,6 @@ async () => {
119
  };
120
  let ort = null;
121
  let tokenizer = null;
122
- let transformersModule = null;
123
- let spanTokenizer = null;
124
- let spanConfig = null;
125
  let sessions = null;
126
  let sessionsProvider = null;
127
  let runtimeProvider = null;
@@ -256,160 +239,14 @@ async () => {
256
  }
257
  async function loadTokenizer() {
258
  if (!tokenizer) {
259
- transformersModule = await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.2");
260
  // Let Transformers.js persist tokenizer assets in its browser cache when supported.
261
- transformersModule.env.useBrowserCache = true;
262
  // Use the Transformers.js-compatible T5 tokenizer repository.
263
- tokenizer = await transformersModule.AutoTokenizer.from_pretrained("Xenova/t5-base");
264
  }
265
  return tokenizer;
266
  }
267
- async function loadSpanTokenizer() {
268
- if (!transformersModule) {
269
- transformersModule = await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.2");
270
- transformersModule.env.useBrowserCache = true;
271
- }
272
- if (!spanTokenizer) {
273
- // Load the exact tokenizer files from the model repo through our shared
274
- // Cache Storage, then instantiate the tokenizer directly from its JSON.
275
- const tokenizerBase = MODEL_BASE + "/peaframe_tokenizer";
276
- const tokenizerJsonBytes = await fetchCachedBytes(tokenizerBase + "/tokenizer.json", "PEAFrame tokenizer · tokenizer.json");
277
- const tokenizerConfigBytes = await fetchCachedBytes(tokenizerBase + "/tokenizer_config.json", "PEAFrame tokenizer · tokenizer_config.json");
278
- await fetchCachedBytes(tokenizerBase + "/special_tokens_map.json", "PEAFrame tokenizer · special_tokens_map.json");
279
- const tokenizerJson = JSON.parse(new TextDecoder().decode(tokenizerJsonBytes));
280
- const tokenizerConfig = JSON.parse(new TextDecoder().decode(tokenizerConfigBytes));
281
- if (typeof transformersModule.PreTrainedTokenizer !== "function") {
282
- throw new Error("This Transformers.js build does not expose PreTrainedTokenizer.");
283
- }
284
- spanTokenizer = new transformersModule.PreTrainedTokenizer(tokenizerJson, tokenizerConfig);
285
- }
286
- return spanTokenizer;
287
- }
288
- async function loadSpanPredictor(provider) {
289
- if (!sessions) throw new Error("Load the base audio models before span prediction.");
290
- if (sessions.peaframe) return sessions.peaframe;
291
- setStatus("Loading optional span-prediction model…");
292
- setLog("Automatic Span Prediction requires the optional PEAFrame ONNX model and external weights. First download can be several GB; assets are cached in this browser.");
293
- const configBytes = await fetchCachedBytes(MODEL_BASE + "/peaframe_config.json", "PEAFrame configuration");
294
- spanConfig = JSON.parse(new TextDecoder().decode(configBytes));
295
- const session = await createSession("peaframe.onnx", provider);
296
- try {
297
- await loadSpanTokenizer();
298
- } catch (error) {
299
- try { await session.release(); } catch (_) {}
300
- throw new Error("Could not load the PEAFrame tokenizer in this browser. Manual Anchors remain available. Details: " + String(error?.message || error));
301
- }
302
- sessions.peaframe = session;
303
- return session;
304
- }
305
- function sigmoid(value) {
306
- if (value >= 0) { const z = Math.exp(-value); return 1 / (1 + z); }
307
- const z = Math.exp(value); return z / (1 + z);
308
- }
309
- async function predictSpans(samples, prompt, threshold, provider) {
310
- const session = await loadSpanPredictor(provider);
311
- const tok = await loadSpanTokenizer();
312
- const encoded = await tok(prompt, {padding: true, truncation: true, max_length: 512});
313
- const rawIds = encoded.input_ids?.data ?? encoded.input_ids;
314
- const rawMask = encoded.attention_mask?.data ?? encoded.attention_mask;
315
- const ids = Array.from(rawIds, Number);
316
- const attention = Array.from(rawMask, Number);
317
- const expectedSamples = 160000; // fixed PEAFrame ONNX input: ~3.33 s at 48 kHz
318
- const hopLength = Number(spanConfig?.hop_length || 1920);
319
- const sampleRate = Number(spanConfig?.sampling_rate || 48000);
320
- const stride = expectedSamples >> 1;
321
- const probabilityChunks = [];
322
- const starts = [];
323
- const startsToProcess = samples.length <= expectedSamples ? [0] : Array.from({length: Math.ceil((samples.length - expectedSamples) / stride) + 1}, (_, i) => i * stride);
324
- for (let ci = 0; ci < startsToProcess.length; ci++) {
325
- const start = startsToProcess[ci];
326
- const end = Math.min(start + expectedSamples, samples.length);
327
- const chunk = new Float32Array(expectedSamples);
328
- chunk.set(samples.subarray(start, end));
329
- const output = await session.run({
330
- input_ids: int64Tensor(ids, [1, ids.length]),
331
- input_values: floatTensor(chunk, [1, 1, expectedSamples]),
332
- attention_mask: int64Tensor(attention, [1, attention.length])
333
- });
334
- const audioEmbeds = output.audio_embeds || output[session.outputNames[0]];
335
- const textEmbeds = output.text_embeds || output[session.outputNames[1]];
336
- if (!audioEmbeds || !textEmbeds) throw new Error("PEAFrame returned unexpected outputs; expected audio_embeds and text_embeds.");
337
- const frames = audioEmbeds.dims[1];
338
- const dim = audioEmbeds.dims[2];
339
- const ae = audioEmbeds.data, te = textEmbeds.data;
340
- const probabilities = new Float32Array(frames);
341
- for (let frame = 0; frame < frames; frame++) {
342
- let dot = 0;
343
- const offset = frame * dim;
344
- for (let d = 0; d < dim; d++) dot += ae[offset + d] * te[d];
345
- probabilities[frame] = sigmoid(dot * Number(spanConfig?.logit_scale ?? 2.298705816268921) + Number(spanConfig?.logit_bias ?? -10.002328872680664));
346
- }
347
- const usableFrames = Math.min(frames, Math.ceil((end - start) / hopLength));
348
- probabilityChunks.push(probabilities.slice(0, usableFrames));
349
- starts.push(start);
350
- setStatus("Predicting sound spans " + (ci + 1) + "/" + startsToProcess.length + "…");
351
- await new Promise(resolve => setTimeout(resolve, 0));
352
- }
353
- const totalFrames = Math.ceil(samples.length / hopLength);
354
- const merged = new Float32Array(totalFrames);
355
- const counts = new Uint16Array(totalFrames);
356
- for (let ci = 0; ci < probabilityChunks.length; ci++) {
357
- const chunkStart = Math.floor(starts[ci] / hopLength);
358
- const probs = probabilityChunks[ci];
359
- for (let j = 0; j < probs.length && chunkStart + j < totalFrames; j++) {
360
- merged[chunkStart + j] += probs[j];
361
- counts[chunkStart + j]++;
362
- }
363
- }
364
- for (let i = 0; i < merged.length; i++) merged[i] = counts[i] ? merged[i] / counts[i] : 0;
365
- const spans = [];
366
- let active = false, startFrame = 0;
367
- for (let i = 0; i < merged.length; i++) {
368
- const detected = merged[i] > threshold;
369
- if (detected && !active) { startFrame = i; active = true; }
370
- else if (!detected && active) {
371
- spans.push([startFrame * hopLength / sampleRate, i * hopLength / sampleRate]);
372
- active = false;
373
- }
374
- }
375
- if (active) spans.push([startFrame * hopLength / sampleRate, samples.length / sampleRate]);
376
- return spans;
377
- }
378
- function parseManualAnchors(raw, duration) {
379
- const lines = raw.split(/\r?\n/).map(line => line.trim()).filter(Boolean);
380
- const spans = [];
381
- for (let index = 0; index < lines.length; index++) {
382
- const parts = lines[index].split(/[\s,;]+/);
383
- if (parts.length !== 3 || !["+", "-"].includes(parts[0])) {
384
- throw new Error("Invalid manual anchor on line " + (index + 1) + '. Use: + 4.5 7.0 or - 0 3.0.');
385
- }
386
- const start = Number(parts[1]), end = Number(parts[2]);
387
- if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end <= start || end > duration) {
388
- throw new Error("Invalid time range on manual anchor line " + (index + 1) + ". Times must be within the audio duration and end must be greater than start.");
389
- }
390
- spans.push({sign: parts[0], start, end});
391
- }
392
- if (!spans.length) throw new Error("Manual Anchors is enabled, but no valid anchor ranges were entered.");
393
- return spans;
394
- }
395
- function buildAnchorTensors(spans, sequenceLength) {
396
- const ids = [0, 3]; // <null>=0, <pad>=3, as in the Python reference
397
- const alignment = new BigInt64Array(sequenceLength);
398
- alignment.fill(1); // unanchored frames map to <pad>
399
- for (const span of spans) {
400
- const startFrame = Math.max(0, Math.min(sequenceLength, Math.floor(span.start * 48000 / 1920)));
401
- const endFrame = Math.max(0, Math.min(sequenceLength, Math.floor(span.end * 48000 / 1920)));
402
- if (startFrame >= endFrame) continue;
403
- const anchorIndex = ids.length;
404
- alignment.fill(BigInt(anchorIndex), startFrame, endFrame);
405
- ids.push(span.sign === "-" ? 2 : 1);
406
- }
407
- return {
408
- ids: new ort.Tensor("int64", BigInt64Array.from(ids, value => BigInt(value)), [1, ids.length]),
409
- alignment: new ort.Tensor("int64", alignment, [1, sequenceLength])
410
- };
411
- }
412
-
413
  async function fetchCachedBytes(url, label) {
414
  // Cache Storage persists across page reloads. If unavailable or quota is exceeded,
415
  // continue with a normal network download rather than failing inference.
@@ -479,10 +316,6 @@ async () => {
479
  sessions = null;
480
  sessionsProvider = null;
481
  }
482
- spanTokenizer = null;
483
- spanConfig = null;
484
- tokenizer = null;
485
- transformersModule = null;
486
  if ("caches" in window) {
487
  try { await caches.delete(CACHE_NAME); } catch (_) {}
488
  }
@@ -604,10 +437,9 @@ async () => {
604
  maskDims: [1, mask.length]
605
  };
606
  }
607
- async function runDit(noisy, timeValue, audioFeatures, text, sequenceLength, anchors = null) {
608
  const batch = 1;
609
- const anchorIds = anchors?.ids || new ort.Tensor("int64", BigInt64Array.from([0n, 3n]), [1, 2]);
610
- const anchorAlignment = anchors?.alignment || new ort.Tensor("int64", new BigInt64Array(sequenceLength), [1, sequenceLength]);
611
  const mask = new Uint8Array(sequenceLength).fill(1);
612
  const video = new Float32Array(1024 * sequenceLength);
613
  const feeds = {
@@ -617,25 +449,25 @@ async () => {
617
  text_features: floatTensor(text.features, text.featureDims),
618
  text_mask: boolTensor(text.mask, text.maskDims),
619
  masked_video_features: floatTensor(video, [batch, 1024, sequenceLength]),
620
- anchor_ids: anchorIds,
621
- anchor_alignment: anchorAlignment,
622
  audio_pad_mask: boolTensor(mask, [batch, sequenceLength])
623
  };
624
  const outputs = await sessions.dit.run(feeds);
625
  const tensor = outputs.velocity || outputs[sessions.dit.outputNames[0]];
626
  return new Float32Array(tensor.data);
627
  }
628
- async function runOde(initial, audioFeatures, text, steps, sequenceLength, anchors = null) {
629
  let x = new Float32Array(initial);
630
  const dt = 1 / steps;
631
  for (let i = 0; i < steps; i++) {
632
  setStatus("Denoising step " + (i + 1) + "/" + steps + " (1/2)…");
633
  const t = i * dt;
634
- const k1 = await runDit(x, t, audioFeatures, text, sequenceLength, anchors);
635
  const midpoint = new Float32Array(x.length);
636
  for (let j = 0; j < x.length; j++) midpoint[j] = x[j] + (dt / 2) * k1[j];
637
  setStatus("Denoising step " + (i + 1) + "/" + steps + " (2/2)…");
638
- const k2 = await runDit(midpoint, t + dt / 2, audioFeatures, text, sequenceLength, anchors);
639
  for (let j = 0; j < x.length; j++) x[j] += dt * k2[j];
640
  // Yield so status/progress updates can paint between long CPU steps.
641
  await new Promise(resolve => setTimeout(resolve, 0));
@@ -714,11 +546,7 @@ async () => {
714
  if (!Number.isInteger(steps) || steps < 1 || steps > 16) throw new Error("ODE steps must be between 1 and 16.");
715
  const audio = await decodeAudio(file);
716
  const expectedSamples = audio.samples.length;
717
- if (expectedSamples > 48000 * 60) throw new Error("Audio must be 60 seconds or shorter.");
718
- const predictSpansEnabled = $("sam-predict-spans").checked;
719
- const manualAnchorsEnabled = $("sam-manual-anchors-enabled").checked;
720
- const manualSpans = manualAnchorsEnabled ? parseManualAnchors($("sam-manual-anchors").value, expectedSamples / 48000) : [];
721
- const spanThreshold = Number($("sam-span-threshold").value);
722
  setStatus("Loading models for " + deviceLabel + "…");
723
  if (provider === "wasm") {
724
  setLog("CPU mode selected. ONNX Runtime will use WebAssembly in a background worker where supported, keeping the page responsive. This may take longer than GPU inference.");
@@ -750,22 +578,7 @@ async () => {
750
  initial[i] = radius * Math.cos(2 * Math.PI * u2);
751
  if (i + 1 < initial.length) initial[i + 1] = radius * Math.sin(2 * Math.PI * u2);
752
  }
753
- let anchorSpans = [];
754
- if (manualAnchorsEnabled) {
755
- anchorSpans = manualSpans;
756
- setStatus("Applying " + anchorSpans.length + " manual anchor range(s)…");
757
- } else if (predictSpansEnabled) {
758
- setStatus("Detecting target sound spans…");
759
- const detected = await predictSpans(audio.samples, prompt, spanThreshold, provider);
760
- anchorSpans = detected.map(([start, end]) => ({sign: "+", start, end}));
761
- if (anchorSpans.length) {
762
- setStatus("Detected " + anchorSpans.length + " sound span(s). Preparing separation…");
763
- } else {
764
- setStatus("No matching spans detected · continuing without span guidance…");
765
- }
766
- }
767
- const anchors = anchorSpans.length ? buildAnchorTensors(anchorSpans, timeSteps) : null;
768
- const result = await runOde(initial, mixture, text, steps, timeSteps, anchors);
769
  const targetLatent = new Float32Array(latentChannels * timeSteps);
770
  const residualLatent = new Float32Array(latentChannels * timeSteps);
771
  for (let t = 0; t < timeSteps; t++) {
@@ -817,15 +630,6 @@ async () => {
817
  window.addEventListener("resize", () => {
818
  for (const [id, entry] of Object.entries(waveforms)) drawWaveform(id, entry.samples, entry.kind);
819
  });
820
- $("sam-predict-spans").addEventListener("change", () => {
821
- $("sam-span-threshold-wrap").hidden = !$("sam-predict-spans").checked;
822
- });
823
- $("sam-manual-anchors-enabled").addEventListener("change", () => {
824
- $("sam-manual-anchors-wrap").hidden = !$("sam-manual-anchors-enabled").checked;
825
- });
826
- $("sam-span-threshold").addEventListener("input", () => {
827
- $("sam-span-threshold-value").textContent = Number($("sam-span-threshold").value).toFixed(2);
828
- });
829
  $("sam-clear-cache").addEventListener("click", async () => {
830
  const button = $("sam-clear-cache");
831
  button.disabled = true;
 
19
  #sam-browser-app .step{display:grid;place-items:center;flex:0 0 34px;height:34px;border-radius:11px;background:color-mix(in srgb,var(--sam-accent) 14%,transparent);color:var(--sam-accent2);font-weight:800}
20
  #sam-browser-app .muted{opacity:.68;font-size:13px;line-height:1.5}
21
  #sam-browser-app label{display:block;margin:0 0 7px;font-size:13px;font-weight:700}
22
+ #sam-browser-app input[type=file],#sam-browser-app input[type=text],#sam-browser-app input[type=number],#sam-browser-app select{width:100%;min-width:0;padding:12px 13px;border-radius:10px;border:1px solid var(--border-color-primary);background:var(--input-background-fill);color:var(--body-text-color);font:inherit;font-size:14px}
23
  #sam-browser-app input[type=file]{padding:10px}
 
 
 
 
 
 
 
 
 
24
  #sam-browser-app .file-wrap{padding:16px;border:1px dashed var(--border-color-primary);border-radius:14px;background:var(--background-fill-secondary)}
25
  #sam-browser-app .controls{display:grid;grid-template-columns:minmax(0,2fr) minmax(150px,1fr) minmax(110px,.7fr);gap:14px;margin-top:18px}
26
  #sam-browser-app .field{min-width:0}
 
51
  #sam-browser-app .empty-results{padding:24px 14px;border:1px dashed var(--border-color-primary);border-radius:14px;text-align:center;color:var(--body-text-color);opacity:.65;font-size:13px}
52
  #sam-browser-app .privacy{display:flex;gap:10px;align-items:flex-start;padding:12px 14px;border-radius:11px;background:color-mix(in srgb,#35b67a 8%,transparent);font-size:12px;line-height:1.5;margin-top:16px}
53
  #sam-browser-app .privacy svg{flex:0 0 17px;margin-top:1px}
54
+ #sam-browser-app .footer-note{font-size:12px;opacity:.58;text-align:center;margin:18px 0 8px;line-height:1.5}
55
+ #sam-browser-app .credits{font-size:12px;opacity:.7;text-align:center;margin:0 0 28px;line-height:1.8}
56
+ #sam-browser-app .credits a{color:var(--sam-accent2);text-decoration:none;font-weight:650}
57
+ #sam-browser-app .credits a:hover{text-decoration:underline}
58
  @media(max-width:820px){#sam-browser-app .audio-grid{grid-template-columns:1fr}#sam-browser-app .controls{grid-template-columns:1fr 1fr}#sam-browser-app .controls .prompt-field{grid-column:1/-1}}
59
+ @media(max-width:480px){#sam-browser-app .controls{grid-template-columns:1fr}#sam-browser-app .controls .prompt-field{grid-column:auto}#sam-browser-app .actions{align-items:stretch}#sam-browser-app .actions button{width:100%}#sam-browser-app .status{flex-basis:100%}}
60
  </style>
61
  <header class="hero">
62
  <div class="eyebrow">Audio separation studio</div>
63
  <h1>SAM Audio <span style="color:var(--sam-accent2)">Studio</span></h1>
64
+ <p class="subhead">Separate speech, music, or other sounds from a recording using Meta's SAM Audio, running locally right in your browser.<br><br>Once downloaded, the model takes up about 2.7 GB in the browser cache.</p>
65
  <div class="pill"><span class="dot"></span> Local browser processing <span style="opacity:.45">·</span> ONNX Runtime</div>
66
  </header>
67
  <section class="panel">
68
  <div class="section-head"><div class="step">01</div><div><h2>Source & settings</h2><div class="muted">Choose a recording and describe what you want to isolate.</div></div></div>
69
+ <div class="file-wrap"><label for="sam-audio-file">Audio file</label><input id="sam-audio-file" type="file" accept="audio/*,.wav,.mp3,.flac,.ogg,.m4a"><div class="muted" style="margin-top:8px">WAV, MP3, FLAC, OGG, or M4A · Recommended: a short clip for the first run</div></div>
70
  <div class="controls">
71
  <div class="field prompt-field"><label for="sam-prompt">What should be isolated?</label><input id="sam-prompt" type="text" value="a person speaking" placeholder="e.g. a person speaking"></div>
72
  <div class="field"><label for="sam-provider">Processing device</label><select id="sam-provider"><option value="auto" selected>Automatic</option><option value="webgpu">GPU · WebGPU</option><option value="wasm">CPU · WASM</option></select></div>
73
  <div class="field"><label for="sam-steps">ODE steps</label><input id="sam-steps" type="number" min="1" max="16" value="2"></div>
74
  </div>
75
  <div class="muted" style="margin-top:9px">Automatic mode uses CPU on phones/tablets and WebGPU on desktop when available. More ODE steps can improve quality but take longer.</div>
 
 
 
 
 
 
 
 
 
76
  <div class="actions"><button id="sam-run">▶ &nbsp; Separate audio</button><button id="sam-clear-cache" class="secondary" type="button">Clear model cache</button><div id="sam-status-box" class="status" data-state="idle"><span class="status-dot"></span><span id="sam-status" aria-live="polite">Ready when you are</span></div></div>
77
  <div class="privacy"><svg viewBox="0 0 24 24" width="17" height="17" fill="none" stroke="currentColor" stroke-width="2"><rect x="4" y="10" width="16" height="11" rx="2"/><path d="M8 10V7a4 4 0 0 1 8 0v3"/></svg><div><strong>Your audio stays on your device.</strong> Inference runs locally in this browser. The Space hosts the interface and model downloads; your selected audio is not uploaded for processing.</div></div>
78
  </section>
 
84
  <article class="audio-card"><div class="audio-card-head"><span class="audio-title">Residual</span><span class="tag neutral">Everything else</span></div><div class="wave-wrap"><canvas id="sam-residual-wave" aria-label="Residual audio waveform"></canvas><div class="wave-empty" id="sam-residual-empty">Run separation to create the residual</div></div><div class="audio-meta"><span id="sam-residual-duration">Waiting for result</span><span>48 kHz · WAV</span></div><audio id="sam-residual-player" controls preload="metadata"></audio><a class="download" id="sam-residual-download" download="residual.wav" hidden>↓ Download residual.wav</a></article>
85
  </div>
86
  </section>
87
+ <p class="footer-note">First use downloads model files and may take several minutes on CPU.</p>
88
+ <div class="credits">Credits: <a href="https://github.com/facebookresearch/sam-audio" target="_blank" rel="noopener noreferrer">Meta's SAM Audio</a> <span aria-hidden="true">·</span> ONNX-converted models by <a href="https://huggingface.co/matbee/sam-audio-small-onnx" target="_blank" rel="noopener noreferrer">matbee</a></div>
89
  </div>
90
  """
91
 
 
105
  };
106
  let ort = null;
107
  let tokenizer = null;
 
 
 
108
  let sessions = null;
109
  let sessionsProvider = null;
110
  let runtimeProvider = null;
 
239
  }
240
  async function loadTokenizer() {
241
  if (!tokenizer) {
242
+ const transformers = await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.2");
243
  // Let Transformers.js persist tokenizer assets in its browser cache when supported.
244
+ transformers.env.useBrowserCache = true;
245
  // Use the Transformers.js-compatible T5 tokenizer repository.
246
+ tokenizer = await transformers.AutoTokenizer.from_pretrained("Xenova/t5-base");
247
  }
248
  return tokenizer;
249
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
250
  async function fetchCachedBytes(url, label) {
251
  // Cache Storage persists across page reloads. If unavailable or quota is exceeded,
252
  // continue with a normal network download rather than failing inference.
 
316
  sessions = null;
317
  sessionsProvider = null;
318
  }
 
 
 
 
319
  if ("caches" in window) {
320
  try { await caches.delete(CACHE_NAME); } catch (_) {}
321
  }
 
437
  maskDims: [1, mask.length]
438
  };
439
  }
440
+ async function runDit(noisy, timeValue, audioFeatures, text, sequenceLength) {
441
  const batch = 1;
442
+ const zerosInt = new BigInt64Array(sequenceLength);
 
443
  const mask = new Uint8Array(sequenceLength).fill(1);
444
  const video = new Float32Array(1024 * sequenceLength);
445
  const feeds = {
 
449
  text_features: floatTensor(text.features, text.featureDims),
450
  text_mask: boolTensor(text.mask, text.maskDims),
451
  masked_video_features: floatTensor(video, [batch, 1024, sequenceLength]),
452
+ anchor_ids: new ort.Tensor("int64", zerosInt, [batch, sequenceLength]),
453
+ anchor_alignment: new ort.Tensor("int64", new BigInt64Array(sequenceLength), [batch, sequenceLength]),
454
  audio_pad_mask: boolTensor(mask, [batch, sequenceLength])
455
  };
456
  const outputs = await sessions.dit.run(feeds);
457
  const tensor = outputs.velocity || outputs[sessions.dit.outputNames[0]];
458
  return new Float32Array(tensor.data);
459
  }
460
+ async function runOde(initial, audioFeatures, text, steps, sequenceLength) {
461
  let x = new Float32Array(initial);
462
  const dt = 1 / steps;
463
  for (let i = 0; i < steps; i++) {
464
  setStatus("Denoising step " + (i + 1) + "/" + steps + " (1/2)…");
465
  const t = i * dt;
466
+ const k1 = await runDit(x, t, audioFeatures, text, sequenceLength);
467
  const midpoint = new Float32Array(x.length);
468
  for (let j = 0; j < x.length; j++) midpoint[j] = x[j] + (dt / 2) * k1[j];
469
  setStatus("Denoising step " + (i + 1) + "/" + steps + " (2/2)…");
470
+ const k2 = await runDit(midpoint, t + dt / 2, audioFeatures, text, sequenceLength);
471
  for (let j = 0; j < x.length; j++) x[j] += dt * k2[j];
472
  // Yield so status/progress updates can paint between long CPU steps.
473
  await new Promise(resolve => setTimeout(resolve, 0));
 
546
  if (!Number.isInteger(steps) || steps < 1 || steps > 16) throw new Error("ODE steps must be between 1 and 16.");
547
  const audio = await decodeAudio(file);
548
  const expectedSamples = audio.samples.length;
549
+ if (expectedSamples > 48000 * 20) throw new Error("For this first prototype, use audio no longer than 20 seconds.");
 
 
 
 
550
  setStatus("Loading models for " + deviceLabel + "…");
551
  if (provider === "wasm") {
552
  setLog("CPU mode selected. ONNX Runtime will use WebAssembly in a background worker where supported, keeping the page responsive. This may take longer than GPU inference.");
 
578
  initial[i] = radius * Math.cos(2 * Math.PI * u2);
579
  if (i + 1 < initial.length) initial[i + 1] = radius * Math.sin(2 * Math.PI * u2);
580
  }
581
+ const result = await runOde(initial, mixture, text, steps, timeSteps);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
582
  const targetLatent = new Float32Array(latentChannels * timeSteps);
583
  const residualLatent = new Float32Array(latentChannels * timeSteps);
584
  for (let t = 0; t < timeSteps; t++) {
 
630
  window.addEventListener("resize", () => {
631
  for (const [id, entry] of Object.entries(waveforms)) drawWaveform(id, entry.samples, entry.kind);
632
  });
 
 
 
 
 
 
 
 
 
633
  $("sam-clear-cache").addEventListener("click", async () => {
634
  const button = $("sam-clear-cache");
635
  button.disabled = true;