import * as Tone from "tone"; import { WorkletSynthesizer } from "spessasynth_lib"; import workletUrl from "spessasynth_lib/dist/spessasynth_processor.min.js?url"; /** * Playback goes through spessasynth_lib, a full SoundFont synthesizer running * in an AudioWorklet, fed with MuseScore_General.sf3 (the vorbis-compressed * build of the same soundfont the backend's fluidsynth /auralize uses). The * backend serves it at /soundfonts/ from a locally-cached download — see * muscriptor/soundfonts.py. */ const SOUNDFONT_URL = "/soundfonts/MuseScore_General.sf3"; /** GM channel reserved for percussion. */ const DRUM_CHANNEL = 9; /** Map a muscriptor instrument-group name to a General MIDI program number. */ const GM_PROGRAM: Record = { acoustic_piano: 0, electric_piano: 4, chromatic_percussion: 9, organ: 19, acoustic_guitar: 24, clean_electric_guitar: 27, distorted_electric_guitar: 30, acoustic_bass: 32, electric_bass: 33, violin: 40, viola: 41, cello: 42, contrabass: 43, orchestral_harp: 46, timpani: 47, string_ensemble: 48, synth_strings: 50, voice: 52, orchestra_hit: 55, trumpet: 56, trombone: 57, tuba: 58, french_horn: 60, brass_section: 61, soprano_and_alto_sax: 65, tenor_sax: 66, baritone_sax: 67, oboe: 68, english_horn: 69, bassoon: 70, clarinet: 71, flutes: 73, synth_lead: 80, synth_pad: 89, }; /** Velocity used for every synthesized note. */ const NOTE_VELOCITY = 100; type NoteOpts = { instrument: string; pitch: number; start: number; end: number; }; type SynthMode = "loading" | "soundfont" | "fallback"; type FallbackOscillator = "sine" | "triangle" | "sawtooth" | "square"; /** * AudioWorklet is restricted to secure contexts, so the plain-HTTP LAN demo * cannot use SpessaSynth. Keep that deployment useful with a lightweight * Tone.js oscillator voice whose timbre roughly follows the predicted family. */ function fallbackOscillator(instrument: string): FallbackOscillator { if (instrument === "drums" || instrument === "timpani") return "square"; if (instrument.includes("guitar") || instrument.includes("bass")) return "sawtooth"; if (instrument.includes("sax") || instrument.includes("brass")) return "sawtooth"; if (instrument === "voice" || instrument.includes("flute")) return "sine"; return "triangle"; } export class AudioEngine { private ctx!: AudioContext; private synth: WorkletSynthesizer | null = null; private synthMode: SynthMode = "loading"; /** Secure-context-independent voices used when AudioWorklet is unavailable. */ private fallbackSynths = new Map(); /** Notes scheduled while the synth / soundfont was still loading. */ private pendingNotes: NoteOpts[] = []; /** MIDI channel assigned to each instrument group. */ private channels = new Map(); /** Zero-gain shadow channel per instrument, used only to pre-decode samples. */ private warmChannels = new Map(); /** `instrument:pitch` pairs already warmed (the synth caches per-pitch voices). */ private warmedNotes = new Set(); private nextChannel = 0; /** All notes ever scheduled — kept so Play-from-0 can re-schedule them. */ private allNotes: NoteOpts[] = []; private autoStopAt: number | null = null; /** Decoded original audio + live playback node, for the WAV/MIDI mix. */ private wavBuffer: AudioBuffer | null = null; private wavSource: AudioBufferSourceNode | null = null; private wavGain: GainNode; private midiGain: GainNode; /** Panners on each bus: centered in mix mode, hard L/R in stereo mode. */ private wavPanner: StereoPannerNode; private midiPanner: StereoPannerNode; private mutedInstruments = new Set(); private mix = 0.75; // 0 = full WAV, 1 = full MIDI /** When true: original audio hard-left, synthesis hard-right (mix ignored). */ private stereo = false; private soundfontUrl: string; constructor(options: { forceFallback?: boolean; soundfontUrl?: string } = {}) { this.soundfontUrl = options.soundfontUrl ?? SOUNDFONT_URL; // Tone.getContext() lazily creates a (suspended) AudioContext — no user // gesture needed. The synth and soundfont can load while suspended; // we only need to call Tone.start() to *resume* it on the play click. this.ctx = Tone.getContext().rawContext as AudioContext; // Each bus runs gain → panner → destination so we can both crossfade // (gain) and place the two sources L/R (pan) independently. this.midiPanner = this.ctx.createStereoPanner(); this.midiPanner.connect(this.ctx.destination); this.midiGain = this.ctx.createGain(); this.midiGain.connect(this.midiPanner); this.wavPanner = this.ctx.createStereoPanner(); this.wavPanner.connect(this.ctx.destination); this.wavGain = this.ctx.createGain(); this.wavGain.connect(this.wavPanner); // AudioWorkletNode is deliberately unavailable on plain-HTTP non-localhost // pages. Those are exactly how the workstation demos are shared, so avoid // downloading a 40 MB SoundFont that cannot be played and select the // oscillator fallback immediately. HTTPS and localhost retain the richer // SoundFont path. if (!options.forceFallback && globalThis.isSecureContext && this.ctx.audioWorklet) { this.initSynth().catch((e) => this.initFallback(e)); } else { this.initFallback(new Error("AudioWorklet requires HTTPS or localhost")); } this.applyMix(); Tone.getTransport().on("start", (time) => this.startWavSource(time)); Tone.getTransport().on("stop", () => this.stopWavSource()); // Pause keeps the transport position but emits "pause", not "stop", so the // raw WAV buffer source must be torn down here too; "start" re-creates it at // the paused offset when playback resumes. Tone.getTransport().on("pause", () => this.stopWavSource()); } /** Load the worklet + soundfont, then flush notes queued in the meantime. */ private async initSynth() { // Fetch the soundfont concurrently with the worklet setup. const soundfont = fetch(this.soundfontUrl).then((res) => { if (!res.ok) throw new Error(`${this.soundfontUrl}: HTTP ${res.status}`); return res.arrayBuffer(); }); // Tone wraps its AudioContext with standardized-audio-context, so the // worklet module and node must be created through Tone's own helpers — // spessasynth's audioNodeCreators hook exists for exactly this. const toneCtx = Tone.getContext(); await toneCtx.addAudioWorkletModule(workletUrl); const synth = new WorkletSynthesizer(this.ctx, { audioNodeCreators: { worklet: (_ctx, name, options) => toneCtx.createAudioWorkletNode(name, options), }, }); synth.connect(this.midiGain); await synth.isReady; await synth.soundBankManager.addSoundBank( await soundfont, "MuseScore_General", ); this.synth = synth; this.synthMode = "soundfont"; this.flushPendingNotes(); } /** Select the basic synth used by insecure-context demo deployments. */ private initFallback(reason: unknown) { if (this.synthMode !== "loading") return; console.warn( "SoundFont synthesis is unavailable; using the basic browser synth instead.", reason, ); this.synthMode = "fallback"; this.flushPendingNotes(); } private flushPendingNotes() { const queued = this.pendingNotes; this.pendingNotes = []; for (const n of queued) this.scheduleNoteRaw(n); } /** Lazily allocate one polyphonic fallback voice per instrument family. */ private fallbackSynthFor(instrument: string): Tone.PolySynth { let voice = this.fallbackSynths.get(instrument); if (voice) return voice; const percussive = instrument === "drums" || instrument === "timpani"; voice = new Tone.PolySynth(Tone.Synth, { oscillator: { type: fallbackOscillator(instrument) }, envelope: percussive ? { attack: 0.001, decay: 0.08, sustain: 0.05, release: 0.05 } : { attack: 0.01, decay: 0.1, sustain: 0.55, release: 0.12 }, }); voice.maxPolyphony = 32; voice.volume.value = this.mutedInstruments.has(instrument) ? -Infinity : -10; voice.connect(this.midiGain); this.fallbackSynths.set(instrument, voice); return voice; } private stopSynthVoices() { this.synth?.stopAll(); for (const voice of this.fallbackSynths.values()) voice.releaseAll(); } /** Decode `file` and remember the buffer for synced WAV playback. */ async loadWav(file: File) { this.stopWavSource(); this.wavBuffer = null; const bytes = await file.arrayBuffer(); try { this.wavBuffer = await this.ctx.decodeAudioData(bytes); } catch { // Non-decodable input — keep MIDI-only playback. this.wavBuffer = null; } // The stereo-mode WAV level depends on the buffer's channel count. this.applyMix(); } /** Set the MIDI/WAV crossfade. `midiAmount` in [0, 1]. */ setMix(midiAmount: number) { this.mix = Math.max(0, Math.min(1, midiAmount)); this.applyMix(); } /** Toggle stereo split: original → left, synthesis → right. Overrides mix. */ setStereo(enabled: boolean) { this.stereo = enabled; this.applyMix(); } private applyMix() { const t = this.ctx.currentTime; if (this.stereo) { // Panned hard to opposite channels. Hard-panning a stereo signal sums // its L+R into one channel (up to +6 dB), so halve the gain of stereo // sources to stay at roughly the level they have in mix mode. Mono // sources pass through a hard pan at unity, so they need no trim. const wavLevel = this.wavBuffer?.numberOfChannels === 1 ? 1 : 0.5; this.wavGain.gain.setTargetAtTime(wavLevel, t, 0.01); this.midiGain.gain.setTargetAtTime(0.5, t, 0.01); // synth is stereo this.wavPanner.pan.setTargetAtTime(-1, t, 0.01); this.midiPanner.pan.setTargetAtTime(1, t, 0.01); } else { // Centered crossfade between the two buses. this.wavGain.gain.setTargetAtTime(1 - this.mix, t, 0.01); this.midiGain.gain.setTargetAtTime(this.mix, t, 0.01); this.wavPanner.pan.setTargetAtTime(0, t, 0.01); this.midiPanner.pan.setTargetAtTime(0, t, 0.01); } } /** Get (or assign) the MIDI channel for an instrument group. */ private channelFor(instrument: string): number { let ch = this.channels.get(instrument); if (ch !== undefined) return ch; const synth = this.synth!; if (instrument === "drums") { ch = DRUM_CHANNEL; } else { ch = this.nextChannel++; if (this.nextChannel === DRUM_CHANNEL) this.nextChannel++; while (ch >= synth.channelCount) synth.addNewChannel(); synth.programChange(ch, GM_PROGRAM[instrument] ?? 0); } if (this.mutedInstruments.has(instrument)) { synth.midiChannels[ch]?.setSystemParameter("isMuted", true); } this.channels.set(instrument, ch); return ch; } /** * Get (or create) the zero-gain shadow channel used to pre-warm `instrument`. * * The soundfont is vorbis-compressed (SF3): the worklet decodes each sample * lazily — synchronously, on the audio thread — the first time a voice needs * it, which audibly stutters playback (see {@link warmNote}). A shadow * channel with the same program but channel gain 0 lets us trigger those * decodes silently; the decoded samples and built voices land in synth-wide * caches keyed by (preset, key, velocity), so the real channel then plays * warm. */ private warmChannelFor(instrument: string): number { let ch = this.warmChannels.get(instrument); if (ch !== undefined) return ch; const synth = this.synth!; ch = this.nextChannel++; if (this.nextChannel === DRUM_CHANNEL) this.nextChannel++; while (ch >= synth.channelCount) synth.addNewChannel(); if (instrument === "drums") { synth.midiChannels[ch]?.setDrums(true); } else { synth.programChange(ch, GM_PROGRAM[instrument] ?? 0); } synth.midiChannels[ch]?.setSystemParameter("gain", 0); this.warmChannels.set(instrument, ch); return ch; } /** * Pre-decode the sample(s) behind a note by playing it once, immediately and * silently, on the instrument's shadow channel. Called as notes stream in * during transcription so first playback doesn't stall on vorbis decodes. * Velocity must match playback ({@link NOTE_VELOCITY}) — the synth's voice * cache is keyed on it. */ private warmNote(instrument: string, pitch: number) { const key = `${instrument}:${pitch}`; if (this.warmedNotes.has(key)) return; this.warmedNotes.add(key); const synth = this.synth!; const ch = this.warmChannelFor(instrument); synth.noteOn(ch, pitch, NOTE_VELOCITY); synth.noteOff(ch, pitch); } /** Mute or unmute a single instrument on the MIDI track. Works live. */ setInstrumentMuted(instrument: string, muted: boolean) { if (muted) this.mutedInstruments.add(instrument); else this.mutedInstruments.delete(instrument); const ch = this.channels.get(instrument); if (ch !== undefined) { this.synth?.midiChannels[ch]?.setSystemParameter("isMuted", muted); } const fallback = this.fallbackSynths.get(instrument); if (fallback) fallback.volume.value = muted ? -Infinity : -10; } private startWavSource(at: number) { this.stopWavSource(); if (!this.wavBuffer) return; const offset = Tone.getTransport().seconds; if (offset >= this.wavBuffer.duration) return; const src = this.ctx.createBufferSource(); src.buffer = this.wavBuffer; src.connect(this.wavGain); src.start(at, offset); this.wavSource = src; } private stopWavSource() { if (!this.wavSource) return; try { this.wavSource.stop(); } catch { // already stopped } this.wavSource.disconnect(); this.wavSource = null; } /** Schedule a note at transport time `start` for `duration` seconds. */ scheduleNote(opts: NoteOpts) { this.allNotes.push({ ...opts }); this.scheduleNoteRaw(opts); } private scheduleNoteRaw(opts: NoteOpts) { if (this.synthMode === "loading") { this.pendingNotes.push(opts); return; } const duration = Math.max(0.05, opts.end - opts.start); if (this.synthMode === "fallback") { const voice = this.fallbackSynthFor(opts.instrument); Tone.getTransport().scheduleOnce((time) => { voice.triggerAttackRelease( Tone.Frequency(opts.pitch, "midi").toFrequency(), duration, time, NOTE_VELOCITY / 127, ); }, opts.start); return; } const synth = this.synth!; const channel = this.channelFor(opts.instrument); this.warmNote(opts.instrument, opts.pitch); // The callback fires slightly ahead of the audio time; passing `time` // through lets the worklet apply both events sample-accurately. Pairing // the noteOff with its noteOn also guarantees no stuck notes. Tone.getTransport().scheduleOnce((time) => { synth.noteOn(channel, opts.pitch, NOTE_VELOCITY, { time }); synth.noteOff(channel, opts.pitch, { time: time + duration }); }, opts.start); } /** Resume the AudioContext. Must be called from a user-gesture handler. */ async unlock() { await Tone.start(); } async play() { await Tone.start(); const t = Tone.getTransport(); // If we're at the very start, every previously-fired scheduleOnce is gone; // re-schedule every known note (and the auto-stop) from scratch. if (t.seconds < 0.001) { t.cancel(); for (const n of this.allNotes) this.scheduleNoteRaw(n); if (this.autoStopAt !== null) this.scheduleStopRaw(this.autoStopAt); } t.start(); } pause() { Tone.getTransport().pause(); this.stopSynthVoices(); } /** Jump the transport to `seconds`, re-scheduling notes from that point. */ seek(seconds: number) { const t = Tone.getTransport(); const target = Math.max(0, seconds); const wasPlaying = t.state === "started"; if (wasPlaying) t.pause(); this.stopSynthVoices(); // scheduleOnce events are one-shot, so a jump (especially backwards) needs // a full re-schedule of everything that starts after the new position. t.cancel(); t.seconds = target; for (const n of this.allNotes) { if (n.start >= target) this.scheduleNoteRaw(n); } if (this.autoStopAt !== null && this.autoStopAt > target) { this.scheduleStopRaw(this.autoStopAt); } // Restarting the transport re-fires the "start" handler, which restarts // the WAV source at the new offset. if (wasPlaying) t.start(); } /** Move the transport clock only — a cheap per-frame seek for live * scrubbing. Call while paused; the one-shot note schedule is left stale, * so follow up with a real seek() before resuming playback. */ scrubTo(seconds: number) { Tone.getTransport().seconds = Math.max(0, seconds); } stop() { const t = Tone.getTransport(); t.stop(); t.cancel(); t.seconds = 0; this.stopSynthVoices(); } /** * Replace the scheduled MIDI note set in place, keeping the loaded original * audio and the current transport position. Used by the side-by-side compare * view to A/B between the two models' transcriptions over one shared * transport — Tone's transport is global, so a second AudioEngine can't run * alongside this one. */ loadNotes(notes: NoteOpts[]) { const t = Tone.getTransport(); const from = t.seconds; t.cancel(); this.stopSynthVoices(); this.allNotes = notes.map((n) => ({ ...n })); this.pendingNotes = []; for (const n of this.allNotes) { if (n.start >= from) this.scheduleNoteRaw(n); } const maxEnd = this.allNotes.reduce((m, n) => Math.max(m, n.end), 0); this.autoStopAt = maxEnd > 0 ? maxEnd + 0.3 : null; if (this.autoStopAt !== null && this.autoStopAt > from) { this.scheduleStopRaw(this.autoStopAt); } } /** Wipe scheduled state for a brand-new transcription. */ reset() { this.stop(); this.allNotes = []; this.pendingNotes = []; this.autoStopAt = null; this.stopWavSource(); this.wavBuffer = null; // The instrument list is rebuilt from scratch, so unmute everything. this.mutedInstruments.clear(); for (const ch of this.channels.values()) { this.synth?.midiChannels[ch]?.setSystemParameter("isMuted", false); } for (const voice of this.fallbackSynths.values()) voice.volume.value = -10; } /** Remember the auto-stop time and schedule it for the current run. */ scheduleStop(t: number) { this.autoStopAt = t; this.scheduleStopRaw(t); } private scheduleStopRaw(at: number) { Tone.getTransport().scheduleOnce(() => { Tone.getTransport().stop(); Tone.getTransport().seconds = 0; this.stopSynthVoices(); }, at); } /** Current playback position in seconds. */ get seconds(): number { return Tone.getTransport().seconds; } /** Duration of the decoded source audio in seconds, or 0 if not yet loaded. */ get duration(): number { return this.wavBuffer?.duration ?? 0; } get state(): "started" | "stopped" | "paused" { return Tone.getTransport().state; } get synthesisMode(): SynthMode { return this.synthMode; } }