// basicpitch.js - Spotify's Basic Pitch in the browser: audio in, notes and MIDI out. // // const output = await transcribe(samples); // mono Float32Array at SAMPLE_RATE // const notes = findNotes(output); // [{ pitchMidi, startTimeSeconds, durationSeconds, amplitude }] // const midi = toMidi(notes); // the bytes of a .mid file import { notesFromOutput } from "./notes.js"; export const SAMPLE_RATE = 22050; const NOTES = 88, FPS = Math.floor(SAMPLE_RATE / 256); const WINDOW = 43844, FRAMES = 172, EDGE = 15, ROWS = FRAMES - 2 * EDGE; // as in bp.c const OVERLAP = 2 * EDGE * 256, STEP = WINDOW - OVERLAP; let pool = null, jobs = 0; // Starts the workers (transcribe does too). The first call sets the number of threads. export function preload({ threads = Math.min(8, navigator.hardwareConcurrency || 4) } = {}) { if (pool) return pool.ready; const workers = Array.from({ length: threads }, () => new Worker(import.meta.url, { type: "module", name: "basicpitch" })); const handlers = new Map(); // job id -> its result handler const ready = loadFiles().then((files) => Promise.all(workers.map((worker) => new Promise((resolve, reject) => { worker.onmessage = ({ data }) => { if (data.ready) resolve(); else if (data.job === undefined) reject(new Error(data.error)); else handlers.get(data.job)?.(worker, data); }; worker.onerror = (e) => { e.preventDefault(); const error = e.message || "Basic Pitch failed"; reject(new Error(error)); for (const handle of handlers.values()) handle(worker, { error }); }; worker.postMessage(files); })))); pool = { workers, handlers, ready }; ready.catch(() => { for (const worker of workers) worker.terminate(); pool = null; }); return ready; } async function loadFiles() { const simd = WebAssembly.validate(new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11])); const get = (name) => fetch(new URL(name, import.meta.url)).then((r) => { if (!r.ok) throw new Error(`can't load ${name} (${r.status})`); return r.arrayBuffer(); }); const [wasm, plan, weights] = await Promise.all([get(simd ? "bp.wasm" : "bp-plain.wasm"), get("plan.bin"), get("weights.bin")]); return { module: await WebAssembly.compile(wasm), plan, weights }; } // The model's output: { frames, onsets, rows }, frames[row * 88 + key] (key 0 is MIDI note 21). export async function transcribe(samples, { threads, onProgress, signal } = {}) { await preload(threads ? { threads } : undefined); const job = ++jobs, rows = Math.floor(samples.length * FPS / SAMPLE_RATE), count = Math.ceil((samples.length + OVERLAP / 2) / STEP); const frames = new Float32Array(rows * NOTES), onsets = new Float32Array(rows * NOTES); let next = 0, done = 0; try { await new Promise((resolve, reject) => { if (signal?.aborted) return reject(new DOMException("Aborted", "AbortError")); signal?.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")), { once: true }); const send = (worker) => { if (next >= count || signal?.aborted) return; const index = next++, start = index * STEP - OVERLAP / 2, window = new Float32Array(WINDOW); window.set(samples.subarray(Math.max(0, start), Math.min(samples.length, start + WINDOW)), Math.max(0, -start)); worker.postMessage({ job, index, window }, [window.buffer]); }; pool.handlers.set(job, (worker, data) => { if (data.error) return reject(new Error(data.error)); const at = data.index * ROWS, n = Math.min(ROWS, rows - at) * NOTES; if (n > 0) { frames.set(data.frames.subarray(0, n), at * NOTES); onsets.set(data.onsets.subarray(0, n), at * NOTES); } done++; // not inside onProgress?.(), which skips its argument without a callback onProgress?.(done / count); if (done === count) resolve(); else send(worker); }); for (const worker of pool.workers) { send(worker); send(worker); } }); } finally { pool.handlers.delete(job); } return { frames, onsets, rows }; } // The settings and defaults are basicpitch.spotify.com's (ms, Hz, 0 for no limit). export function findNotes(output, { onsetThreshold = 0.5, frameThreshold = 0.3, minNoteLength = 128, minFrequency = 0, maxFrequency = 3000, boostOnsets = true } = {}) { return notesFromOutput(output, onsetThreshold, frameThreshold, Math.round(minNoteLength / 1000 * FPS), minFrequency, maxFrequency, boostOnsets); } // The notes as the MIDI file holds them (ticks, seconds, velocity), for playback. export function midiNotes(notes, { tempo = 120 } = {}) { const ticks = 8 * tempo; // a second, at 480 a beat return notes.map((n) => { const on = Math.round(n.startTimeSeconds * ticks), off = Math.max(on + 1, Math.round((n.startTimeSeconds + n.durationSeconds) * ticks)); return { pitch: n.pitchMidi, on, off, start: on / ticks, end: off / ticks, velocity: Math.min(127, Math.max(1, Math.round(n.amplitude * 127))) }; }); } // The tempo only sets the beat grid, the notes keep their times. export function toMidi(notes, { tempo = 120 } = {}) { const events = []; for (const n of midiNotes(notes, { tempo })) events.push([n.on, 1, n.pitch, n.velocity], [n.off, 0, n.pitch, 0]); events.sort((a, b) => a[0] - b[0] || a[1] - b[1]); // offs before ons const us = Math.floor(60e6 / tempo); const track = [0x00, 0xff, 0x51, 0x03, (us >> 16) & 255, (us >> 8) & 255, us & 255]; let last = 0; for (const [tick, on, pitch, velocity] of events) { track.push(...varint(tick - last), on ? 0x90 : 0x80, pitch, velocity); last = tick; } track.push(0x00, 0xff, 0x2f, 0x00); const n = track.length; return new Uint8Array([ 0x4d, 0x54, 0x68, 0x64, 0, 0, 0, 6, 0, 0, 0, 1, 0x01, 0xe0, // MThd: format 0, 1 track, 480 ticks a beat 0x4d, 0x54, 0x72, 0x6b, n >>> 24, (n >> 16) & 255, (n >> 8) & 255, n & 255, ...track, ]); } function varint(value) { const bytes = [value & 0x7f]; while ((value >>= 7)) bytes.unshift((value & 0x7f) | 0x80); return bytes; } // In a worker let model = null; async function setUp({ module, plan, weights }) { const bp = (await WebAssembly.instantiate(module, { env: { emscripten_notify_memory_growth() {} } })).exports; bp._initialize?.(); const copy = (bytes) => { const p = bp.malloc(bytes.byteLength); new Uint8Array(bp.memory.buffer, p, bytes.byteLength).set(new Uint8Array(bytes)); return p; }; const handle = bp.bp_load(copy(plan), plan.byteLength / 4, copy(weights)); if (!handle) throw new Error("plan.bin is not a Basic Pitch plan"); model = { bp, handle, input: bp.malloc(WINDOW * 4), frames: bp.malloc(FRAMES * NOTES * 4), onsets: bp.malloc(FRAMES * NOTES * 4) }; // warm up, so the browser compiles the optimised code before real work for (let i = 0; i < 2; i++) bp.bp_run(handle, model.input, model.frames, model.onsets, 0); } function run({ job, index, window }) { const { bp, handle, input, frames, onsets } = model; new Float32Array(bp.memory.buffer, input, WINDOW).set(window); const err = bp.bp_run(handle, input, frames, onsets, 0); if (err) throw new Error(`Basic Pitch error ${err}`); const rows = (p) => new Float32Array(bp.memory.buffer, p + EDGE * NOTES * 4, ROWS * NOTES).slice(); const f = rows(frames), o = rows(onsets); self.postMessage({ job, index, frames: f, onsets: o }, [f.buffer, o.buffer]); } if (typeof WorkerGlobalScope !== "undefined" && self.name === "basicpitch") self.onmessage = ({ data }) => { if (data.module) setUp(data).then(() => self.postMessage({ ready: true }), (error) => self.postMessage({ error: error.message })); else try { run(data); } catch (error) { self.postMessage({ job: data.job, error: error.message }); } };