Download source/demo/web/src/audio.ts from jackboyla/hendrix: direct link, hf CLI and curl.
- Browser
- Download file 19.5 kB
-
https://huggingface.co/spaces/jackboyla/hendrix/resolve/main/source/demo/web/src/audio.ts
- Command line
-
hf download hf://spaces/jackboyla/hendrix/source/demo/web/src/audio.ts
-
curl -L -o audio.ts https://huggingface.co/spaces/jackboyla/hendrix/resolve/main/source/demo/web/src/audio.ts
19.5 kB
| import * as Tone from "tone"; | |
| import { WorkletSynthesizer } from "spessasynth_lib"; | |
| import workletUrl from "spessasynth_lib/dist/spessasynth_processor.min.js?url"; | |
| /** | |
| * Playback goes through spessasynth_lib, a full SoundFont synthesizer running | |
| * in an AudioWorklet, fed with MuseScore_General.sf3 (the vorbis-compressed | |
| * build of the same soundfont the backend's fluidsynth /auralize uses). The | |
| * backend serves it at /soundfonts/ from a locally-cached download β see | |
| * muscriptor/soundfonts.py. | |
| */ | |
| const SOUNDFONT_URL = "/soundfonts/MuseScore_General.sf3"; | |
| /** GM channel reserved for percussion. */ | |
| const DRUM_CHANNEL = 9; | |
| /** Map a muscriptor instrument-group name to a General MIDI program number. */ | |
| const GM_PROGRAM: Record<string, number> = { | |
| acoustic_piano: 0, | |
| electric_piano: 4, | |
| chromatic_percussion: 9, | |
| organ: 19, | |
| acoustic_guitar: 24, | |
| clean_electric_guitar: 27, | |
| distorted_electric_guitar: 30, | |
| acoustic_bass: 32, | |
| electric_bass: 33, | |
| violin: 40, | |
| viola: 41, | |
| cello: 42, | |
| contrabass: 43, | |
| orchestral_harp: 46, | |
| timpani: 47, | |
| string_ensemble: 48, | |
| synth_strings: 50, | |
| voice: 52, | |
| orchestra_hit: 55, | |
| trumpet: 56, | |
| trombone: 57, | |
| tuba: 58, | |
| french_horn: 60, | |
| brass_section: 61, | |
| soprano_and_alto_sax: 65, | |
| tenor_sax: 66, | |
| baritone_sax: 67, | |
| oboe: 68, | |
| english_horn: 69, | |
| bassoon: 70, | |
| clarinet: 71, | |
| flutes: 73, | |
| synth_lead: 80, | |
| synth_pad: 89, | |
| }; | |
| /** Velocity used for every synthesized note. */ | |
| const NOTE_VELOCITY = 100; | |
| type NoteOpts = { | |
| instrument: string; | |
| pitch: number; | |
| start: number; | |
| end: number; | |
| }; | |
| type SynthMode = "loading" | "soundfont" | "fallback"; | |
| type FallbackOscillator = "sine" | "triangle" | "sawtooth" | "square"; | |
| /** | |
| * AudioWorklet is restricted to secure contexts, so the plain-HTTP LAN demo | |
| * cannot use SpessaSynth. Keep that deployment useful with a lightweight | |
| * Tone.js oscillator voice whose timbre roughly follows the predicted family. | |
| */ | |
| function fallbackOscillator(instrument: string): FallbackOscillator { | |
| if (instrument === "drums" || instrument === "timpani") return "square"; | |
| if (instrument.includes("guitar") || instrument.includes("bass")) return "sawtooth"; | |
| if (instrument.includes("sax") || instrument.includes("brass")) return "sawtooth"; | |
| if (instrument === "voice" || instrument.includes("flute")) return "sine"; | |
| return "triangle"; | |
| } | |
| export class AudioEngine { | |
| private ctx!: AudioContext; | |
| private synth: WorkletSynthesizer | null = null; | |
| private synthMode: SynthMode = "loading"; | |
| /** Secure-context-independent voices used when AudioWorklet is unavailable. */ | |
| private fallbackSynths = new Map<string, Tone.PolySynth>(); | |
| /** Notes scheduled while the synth / soundfont was still loading. */ | |
| private pendingNotes: NoteOpts[] = []; | |
| /** MIDI channel assigned to each instrument group. */ | |
| private channels = new Map<string, number>(); | |
| /** Zero-gain shadow channel per instrument, used only to pre-decode samples. */ | |
| private warmChannels = new Map<string, number>(); | |
| /** `instrument:pitch` pairs already warmed (the synth caches per-pitch voices). */ | |
| private warmedNotes = new Set<string>(); | |
| private nextChannel = 0; | |
| /** All notes ever scheduled β kept so Play-from-0 can re-schedule them. */ | |
| private allNotes: NoteOpts[] = []; | |
| private autoStopAt: number | null = null; | |
| /** Decoded original audio + live playback node, for the WAV/MIDI mix. */ | |
| private wavBuffer: AudioBuffer | null = null; | |
| private wavSource: AudioBufferSourceNode | null = null; | |
| private wavGain: GainNode; | |
| private midiGain: GainNode; | |
| /** Panners on each bus: centered in mix mode, hard L/R in stereo mode. */ | |
| private wavPanner: StereoPannerNode; | |
| private midiPanner: StereoPannerNode; | |
| private mutedInstruments = new Set<string>(); | |
| private mix = 0.75; // 0 = full WAV, 1 = full MIDI | |
| /** When true: original audio hard-left, synthesis hard-right (mix ignored). */ | |
| private stereo = false; | |
| private soundfontUrl: string; | |
| constructor(options: { forceFallback?: boolean; soundfontUrl?: string } = {}) { | |
| this.soundfontUrl = options.soundfontUrl ?? SOUNDFONT_URL; | |
| // Tone.getContext() lazily creates a (suspended) AudioContext β no user | |
| // gesture needed. The synth and soundfont can load while suspended; | |
| // we only need to call Tone.start() to *resume* it on the play click. | |
| this.ctx = Tone.getContext().rawContext as AudioContext; | |
| // Each bus runs gain β panner β destination so we can both crossfade | |
| // (gain) and place the two sources L/R (pan) independently. | |
| this.midiPanner = this.ctx.createStereoPanner(); | |
| this.midiPanner.connect(this.ctx.destination); | |
| this.midiGain = this.ctx.createGain(); | |
| this.midiGain.connect(this.midiPanner); | |
| this.wavPanner = this.ctx.createStereoPanner(); | |
| this.wavPanner.connect(this.ctx.destination); | |
| this.wavGain = this.ctx.createGain(); | |
| this.wavGain.connect(this.wavPanner); | |
| // AudioWorkletNode is deliberately unavailable on plain-HTTP non-localhost | |
| // pages. Those are exactly how the workstation demos are shared, so avoid | |
| // downloading a 40 MB SoundFont that cannot be played and select the | |
| // oscillator fallback immediately. HTTPS and localhost retain the richer | |
| // SoundFont path. | |
| if (!options.forceFallback && globalThis.isSecureContext && this.ctx.audioWorklet) { | |
| this.initSynth().catch((e) => this.initFallback(e)); | |
| } else { | |
| this.initFallback(new Error("AudioWorklet requires HTTPS or localhost")); | |
| } | |
| this.applyMix(); | |
| Tone.getTransport().on("start", (time) => this.startWavSource(time)); | |
| Tone.getTransport().on("stop", () => this.stopWavSource()); | |
| // Pause keeps the transport position but emits "pause", not "stop", so the | |
| // raw WAV buffer source must be torn down here too; "start" re-creates it at | |
| // the paused offset when playback resumes. | |
| Tone.getTransport().on("pause", () => this.stopWavSource()); | |
| } | |
| /** Load the worklet + soundfont, then flush notes queued in the meantime. */ | |
| private async initSynth() { | |
| // Fetch the soundfont concurrently with the worklet setup. | |
| const soundfont = fetch(this.soundfontUrl).then((res) => { | |
| if (!res.ok) throw new Error(`${this.soundfontUrl}: HTTP ${res.status}`); | |
| return res.arrayBuffer(); | |
| }); | |
| // Tone wraps its AudioContext with standardized-audio-context, so the | |
| // worklet module and node must be created through Tone's own helpers β | |
| // spessasynth's audioNodeCreators hook exists for exactly this. | |
| const toneCtx = Tone.getContext(); | |
| await toneCtx.addAudioWorkletModule(workletUrl); | |
| const synth = new WorkletSynthesizer(this.ctx, { | |
| audioNodeCreators: { | |
| worklet: (_ctx, name, options) => | |
| toneCtx.createAudioWorkletNode(name, options), | |
| }, | |
| }); | |
| synth.connect(this.midiGain); | |
| await synth.isReady; | |
| await synth.soundBankManager.addSoundBank( | |
| await soundfont, | |
| "MuseScore_General", | |
| ); | |
| this.synth = synth; | |
| this.synthMode = "soundfont"; | |
| this.flushPendingNotes(); | |
| } | |
| /** Select the basic synth used by insecure-context demo deployments. */ | |
| private initFallback(reason: unknown) { | |
| if (this.synthMode !== "loading") return; | |
| console.warn( | |
| "SoundFont synthesis is unavailable; using the basic browser synth instead.", | |
| reason, | |
| ); | |
| this.synthMode = "fallback"; | |
| this.flushPendingNotes(); | |
| } | |
| private flushPendingNotes() { | |
| const queued = this.pendingNotes; | |
| this.pendingNotes = []; | |
| for (const n of queued) this.scheduleNoteRaw(n); | |
| } | |
| /** Lazily allocate one polyphonic fallback voice per instrument family. */ | |
| private fallbackSynthFor(instrument: string): Tone.PolySynth { | |
| let voice = this.fallbackSynths.get(instrument); | |
| if (voice) return voice; | |
| const percussive = instrument === "drums" || instrument === "timpani"; | |
| voice = new Tone.PolySynth(Tone.Synth, { | |
| oscillator: { type: fallbackOscillator(instrument) }, | |
| envelope: percussive | |
| ? { attack: 0.001, decay: 0.08, sustain: 0.05, release: 0.05 } | |
| : { attack: 0.01, decay: 0.1, sustain: 0.55, release: 0.12 }, | |
| }); | |
| voice.maxPolyphony = 32; | |
| voice.volume.value = this.mutedInstruments.has(instrument) ? -Infinity : -10; | |
| voice.connect(this.midiGain); | |
| this.fallbackSynths.set(instrument, voice); | |
| return voice; | |
| } | |
| private stopSynthVoices() { | |
| this.synth?.stopAll(); | |
| for (const voice of this.fallbackSynths.values()) voice.releaseAll(); | |
| } | |
| /** Decode `file` and remember the buffer for synced WAV playback. */ | |
| async loadWav(file: File) { | |
| this.stopWavSource(); | |
| this.wavBuffer = null; | |
| const bytes = await file.arrayBuffer(); | |
| try { | |
| this.wavBuffer = await this.ctx.decodeAudioData(bytes); | |
| } catch { | |
| // Non-decodable input β keep MIDI-only playback. | |
| this.wavBuffer = null; | |
| } | |
| // The stereo-mode WAV level depends on the buffer's channel count. | |
| this.applyMix(); | |
| } | |
| /** Set the MIDI/WAV crossfade. `midiAmount` in [0, 1]. */ | |
| setMix(midiAmount: number) { | |
| this.mix = Math.max(0, Math.min(1, midiAmount)); | |
| this.applyMix(); | |
| } | |
| /** Toggle stereo split: original β left, synthesis β right. Overrides mix. */ | |
| setStereo(enabled: boolean) { | |
| this.stereo = enabled; | |
| this.applyMix(); | |
| } | |
| private applyMix() { | |
| const t = this.ctx.currentTime; | |
| if (this.stereo) { | |
| // Panned hard to opposite channels. Hard-panning a stereo signal sums | |
| // its L+R into one channel (up to +6 dB), so halve the gain of stereo | |
| // sources to stay at roughly the level they have in mix mode. Mono | |
| // sources pass through a hard pan at unity, so they need no trim. | |
| const wavLevel = this.wavBuffer?.numberOfChannels === 1 ? 1 : 0.5; | |
| this.wavGain.gain.setTargetAtTime(wavLevel, t, 0.01); | |
| this.midiGain.gain.setTargetAtTime(0.5, t, 0.01); // synth is stereo | |
| this.wavPanner.pan.setTargetAtTime(-1, t, 0.01); | |
| this.midiPanner.pan.setTargetAtTime(1, t, 0.01); | |
| } else { | |
| // Centered crossfade between the two buses. | |
| this.wavGain.gain.setTargetAtTime(1 - this.mix, t, 0.01); | |
| this.midiGain.gain.setTargetAtTime(this.mix, t, 0.01); | |
| this.wavPanner.pan.setTargetAtTime(0, t, 0.01); | |
| this.midiPanner.pan.setTargetAtTime(0, t, 0.01); | |
| } | |
| } | |
| /** Get (or assign) the MIDI channel for an instrument group. */ | |
| private channelFor(instrument: string): number { | |
| let ch = this.channels.get(instrument); | |
| if (ch !== undefined) return ch; | |
| const synth = this.synth!; | |
| if (instrument === "drums") { | |
| ch = DRUM_CHANNEL; | |
| } else { | |
| ch = this.nextChannel++; | |
| if (this.nextChannel === DRUM_CHANNEL) this.nextChannel++; | |
| while (ch >= synth.channelCount) synth.addNewChannel(); | |
| synth.programChange(ch, GM_PROGRAM[instrument] ?? 0); | |
| } | |
| if (this.mutedInstruments.has(instrument)) { | |
| synth.midiChannels[ch]?.setSystemParameter("isMuted", true); | |
| } | |
| this.channels.set(instrument, ch); | |
| return ch; | |
| } | |
| /** | |
| * Get (or create) the zero-gain shadow channel used to pre-warm `instrument`. | |
| * | |
| * The soundfont is vorbis-compressed (SF3): the worklet decodes each sample | |
| * lazily β synchronously, on the audio thread β the first time a voice needs | |
| * it, which audibly stutters playback (see {@link warmNote}). A shadow | |
| * channel with the same program but channel gain 0 lets us trigger those | |
| * decodes silently; the decoded samples and built voices land in synth-wide | |
| * caches keyed by (preset, key, velocity), so the real channel then plays | |
| * warm. | |
| */ | |
| private warmChannelFor(instrument: string): number { | |
| let ch = this.warmChannels.get(instrument); | |
| if (ch !== undefined) return ch; | |
| const synth = this.synth!; | |
| ch = this.nextChannel++; | |
| if (this.nextChannel === DRUM_CHANNEL) this.nextChannel++; | |
| while (ch >= synth.channelCount) synth.addNewChannel(); | |
| if (instrument === "drums") { | |
| synth.midiChannels[ch]?.setDrums(true); | |
| } else { | |
| synth.programChange(ch, GM_PROGRAM[instrument] ?? 0); | |
| } | |
| synth.midiChannels[ch]?.setSystemParameter("gain", 0); | |
| this.warmChannels.set(instrument, ch); | |
| return ch; | |
| } | |
| /** | |
| * Pre-decode the sample(s) behind a note by playing it once, immediately and | |
| * silently, on the instrument's shadow channel. Called as notes stream in | |
| * during transcription so first playback doesn't stall on vorbis decodes. | |
| * Velocity must match playback ({@link NOTE_VELOCITY}) β the synth's voice | |
| * cache is keyed on it. | |
| */ | |
| private warmNote(instrument: string, pitch: number) { | |
| const key = `${instrument}:${pitch}`; | |
| if (this.warmedNotes.has(key)) return; | |
| this.warmedNotes.add(key); | |
| const synth = this.synth!; | |
| const ch = this.warmChannelFor(instrument); | |
| synth.noteOn(ch, pitch, NOTE_VELOCITY); | |
| synth.noteOff(ch, pitch); | |
| } | |
| /** Mute or unmute a single instrument on the MIDI track. Works live. */ | |
| setInstrumentMuted(instrument: string, muted: boolean) { | |
| if (muted) this.mutedInstruments.add(instrument); | |
| else this.mutedInstruments.delete(instrument); | |
| const ch = this.channels.get(instrument); | |
| if (ch !== undefined) { | |
| this.synth?.midiChannels[ch]?.setSystemParameter("isMuted", muted); | |
| } | |
| const fallback = this.fallbackSynths.get(instrument); | |
| if (fallback) fallback.volume.value = muted ? -Infinity : -10; | |
| } | |
| private startWavSource(at: number) { | |
| this.stopWavSource(); | |
| if (!this.wavBuffer) return; | |
| const offset = Tone.getTransport().seconds; | |
| if (offset >= this.wavBuffer.duration) return; | |
| const src = this.ctx.createBufferSource(); | |
| src.buffer = this.wavBuffer; | |
| src.connect(this.wavGain); | |
| src.start(at, offset); | |
| this.wavSource = src; | |
| } | |
| private stopWavSource() { | |
| if (!this.wavSource) return; | |
| try { | |
| this.wavSource.stop(); | |
| } catch { | |
| // already stopped | |
| } | |
| this.wavSource.disconnect(); | |
| this.wavSource = null; | |
| } | |
| /** Schedule a note at transport time `start` for `duration` seconds. */ | |
| scheduleNote(opts: NoteOpts) { | |
| this.allNotes.push({ ...opts }); | |
| this.scheduleNoteRaw(opts); | |
| } | |
| private scheduleNoteRaw(opts: NoteOpts) { | |
| if (this.synthMode === "loading") { | |
| this.pendingNotes.push(opts); | |
| return; | |
| } | |
| const duration = Math.max(0.05, opts.end - opts.start); | |
| if (this.synthMode === "fallback") { | |
| const voice = this.fallbackSynthFor(opts.instrument); | |
| Tone.getTransport().scheduleOnce((time) => { | |
| voice.triggerAttackRelease( | |
| Tone.Frequency(opts.pitch, "midi").toFrequency(), | |
| duration, | |
| time, | |
| NOTE_VELOCITY / 127, | |
| ); | |
| }, opts.start); | |
| return; | |
| } | |
| const synth = this.synth!; | |
| const channel = this.channelFor(opts.instrument); | |
| this.warmNote(opts.instrument, opts.pitch); | |
| // The callback fires slightly ahead of the audio time; passing `time` | |
| // through lets the worklet apply both events sample-accurately. Pairing | |
| // the noteOff with its noteOn also guarantees no stuck notes. | |
| Tone.getTransport().scheduleOnce((time) => { | |
| synth.noteOn(channel, opts.pitch, NOTE_VELOCITY, { time }); | |
| synth.noteOff(channel, opts.pitch, { time: time + duration }); | |
| }, opts.start); | |
| } | |
| /** Resume the AudioContext. Must be called from a user-gesture handler. */ | |
| async unlock() { | |
| await Tone.start(); | |
| } | |
| async play() { | |
| await Tone.start(); | |
| const t = Tone.getTransport(); | |
| // If we're at the very start, every previously-fired scheduleOnce is gone; | |
| // re-schedule every known note (and the auto-stop) from scratch. | |
| if (t.seconds < 0.001) { | |
| t.cancel(); | |
| for (const n of this.allNotes) this.scheduleNoteRaw(n); | |
| if (this.autoStopAt !== null) this.scheduleStopRaw(this.autoStopAt); | |
| } | |
| t.start(); | |
| } | |
| pause() { | |
| Tone.getTransport().pause(); | |
| this.stopSynthVoices(); | |
| } | |
| /** Jump the transport to `seconds`, re-scheduling notes from that point. */ | |
| seek(seconds: number) { | |
| const t = Tone.getTransport(); | |
| const target = Math.max(0, seconds); | |
| const wasPlaying = t.state === "started"; | |
| if (wasPlaying) t.pause(); | |
| this.stopSynthVoices(); | |
| // scheduleOnce events are one-shot, so a jump (especially backwards) needs | |
| // a full re-schedule of everything that starts after the new position. | |
| t.cancel(); | |
| t.seconds = target; | |
| for (const n of this.allNotes) { | |
| if (n.start >= target) this.scheduleNoteRaw(n); | |
| } | |
| if (this.autoStopAt !== null && this.autoStopAt > target) { | |
| this.scheduleStopRaw(this.autoStopAt); | |
| } | |
| // Restarting the transport re-fires the "start" handler, which restarts | |
| // the WAV source at the new offset. | |
| if (wasPlaying) t.start(); | |
| } | |
| /** Move the transport clock only β a cheap per-frame seek for live | |
| * scrubbing. Call while paused; the one-shot note schedule is left stale, | |
| * so follow up with a real seek() before resuming playback. */ | |
| scrubTo(seconds: number) { | |
| Tone.getTransport().seconds = Math.max(0, seconds); | |
| } | |
| stop() { | |
| const t = Tone.getTransport(); | |
| t.stop(); | |
| t.cancel(); | |
| t.seconds = 0; | |
| this.stopSynthVoices(); | |
| } | |
| /** | |
| * Replace the scheduled MIDI note set in place, keeping the loaded original | |
| * audio and the current transport position. Used by the side-by-side compare | |
| * view to A/B between the two models' transcriptions over one shared | |
| * transport β Tone's transport is global, so a second AudioEngine can't run | |
| * alongside this one. | |
| */ | |
| loadNotes(notes: NoteOpts[]) { | |
| const t = Tone.getTransport(); | |
| const from = t.seconds; | |
| t.cancel(); | |
| this.stopSynthVoices(); | |
| this.allNotes = notes.map((n) => ({ ...n })); | |
| this.pendingNotes = []; | |
| for (const n of this.allNotes) { | |
| if (n.start >= from) this.scheduleNoteRaw(n); | |
| } | |
| const maxEnd = this.allNotes.reduce((m, n) => Math.max(m, n.end), 0); | |
| this.autoStopAt = maxEnd > 0 ? maxEnd + 0.3 : null; | |
| if (this.autoStopAt !== null && this.autoStopAt > from) { | |
| this.scheduleStopRaw(this.autoStopAt); | |
| } | |
| } | |
| /** Wipe scheduled state for a brand-new transcription. */ | |
| reset() { | |
| this.stop(); | |
| this.allNotes = []; | |
| this.pendingNotes = []; | |
| this.autoStopAt = null; | |
| this.stopWavSource(); | |
| this.wavBuffer = null; | |
| // The instrument list is rebuilt from scratch, so unmute everything. | |
| this.mutedInstruments.clear(); | |
| for (const ch of this.channels.values()) { | |
| this.synth?.midiChannels[ch]?.setSystemParameter("isMuted", false); | |
| } | |
| for (const voice of this.fallbackSynths.values()) voice.volume.value = -10; | |
| } | |
| /** Remember the auto-stop time and schedule it for the current run. */ | |
| scheduleStop(t: number) { | |
| this.autoStopAt = t; | |
| this.scheduleStopRaw(t); | |
| } | |
| private scheduleStopRaw(at: number) { | |
| Tone.getTransport().scheduleOnce(() => { | |
| Tone.getTransport().stop(); | |
| Tone.getTransport().seconds = 0; | |
| this.stopSynthVoices(); | |
| }, at); | |
| } | |
| /** Current playback position in seconds. */ | |
| get seconds(): number { | |
| return Tone.getTransport().seconds; | |
| } | |
| /** Duration of the decoded source audio in seconds, or 0 if not yet loaded. */ | |
| get duration(): number { | |
| return this.wavBuffer?.duration ?? 0; | |
| } | |
| get state(): "started" | "stopped" | "paused" { | |
| return Tone.getTransport().state; | |
| } | |
| get synthesisMode(): SynthMode { | |
| return this.synthMode; | |
| } | |
| } | |