Spaces:
Running
Running
cfahlgren1 HF Staff
convert talking pixel to a sveltekit app that runs pivo-1.4m tts in the browser
3ec6b9e | // Synthesis worker: loads assets, owns the engine, streams PCM chunks to the page. | |
| import { loadAssets } from './assets.ts'; | |
| import { parseLexicon, FrontendError } from './frontend.ts'; | |
| import { PiVoEngine, SAMPLE_RATE, HOP } from './engine.ts'; | |
| import { createLiteBackend } from './backends.ts'; | |
| import type { LiteCache } from './backends.ts'; | |
| import type { WorkerMessage, WorkerRequest } from './protocol.ts'; | |
| const scope = self as unknown as { | |
| postMessage(message: WorkerMessage, transfer?: Transferable[]): void; | |
| onmessage: ((event: MessageEvent<WorkerRequest>) => void) | null; | |
| }; | |
| let engine: PiVoEngine<LiteCache> | null = null; | |
| let loading: Promise<void> | null = null; | |
| let latest = 0; // newest request id; older renders stop at their next chunk | |
| // Yield to the event loop (a macrotask) so newer requests can arrive mid-render. | |
| const channel = new MessageChannel(); | |
| const pending: (() => void)[] = []; | |
| channel.port1.onmessage = () => pending.shift()?.(); | |
| const yieldTask = () => | |
| new Promise<void>((resolve) => { | |
| pending.push(resolve); | |
| channel.port2.postMessage(0); | |
| }); | |
| async function init(baseUrl: string): Promise<void> { | |
| const t0 = performance.now(); | |
| const { buffers, fromCache, networkBytes, totalBytes } = await loadAssets(baseUrl, (p) => scope.postMessage({ type: 'progress', ...p })); | |
| const tLoaded = performance.now(); | |
| const dictionary = parseLexicon(new TextDecoder().decode(buffers.cmudict)); | |
| const backend = createLiteBackend(buffers); | |
| engine = new PiVoEngine(backend, dictionary); | |
| // Warm up JIT / wasm on a tiny utterance (not played). | |
| const warm = await engine.plan('Hi.', { voice: 'Pi' }); | |
| await engine.render(warm); | |
| scope.postMessage({ | |
| type: 'ready', backend: backend.name, fromCache, networkBytes, totalBytes, | |
| loadMs: tLoaded - t0, initMs: performance.now() - tLoaded, words: dictionary.size, | |
| }); | |
| } | |
| async function speak({ id, text, voice }: Extract<WorkerRequest, { type: 'speak' }>): Promise<void> { | |
| latest = id; | |
| try { | |
| await loading; | |
| if (!engine) throw new Error('The voice failed to load.'); | |
| const t0 = performance.now(); | |
| const plan = await engine.plan(text, { voice }); | |
| scope.postMessage({ | |
| type: 'plan', id, frames: plan.frames, normalized: plan.front.normalized, words: plan.words, | |
| symbols: plan.front.symbols, tokenStart: plan.tokenStart, durations: plan.durations, | |
| unknown: plan.front.unknown, sampleRate: SAMPLE_RATE, hop: HOP, | |
| }); | |
| let firstChunkMs: number | null = null; | |
| const audio = await engine.render(plan, { | |
| onFrames: (pcm, firstFrame) => { | |
| if (firstChunkMs === null) firstChunkMs = performance.now() - t0; | |
| const copy = pcm.slice(); | |
| scope.postMessage({ type: 'audio', id, pcm: copy, firstFrame }, [copy.buffer]); | |
| }, | |
| shouldStop: () => latest !== id, | |
| yieldEvery: yieldTask, | |
| }); | |
| if (latest !== id) return; | |
| const computeMs = performance.now() - t0; | |
| scope.postMessage({ type: 'done', id, seconds: audio.length / SAMPLE_RATE, computeMs, firstChunkMs }); | |
| } catch (error) { | |
| const err = error as Error & { characters?: string[] }; | |
| scope.postMessage({ type: 'error', id, message: err.message, user: error instanceof FrontendError, characters: err.characters }); | |
| } | |
| } | |
| scope.onmessage = (event) => { | |
| const msg = event.data; | |
| switch (msg.type) { | |
| case 'init': | |
| loading = init(msg.baseUrl).catch((error: Error) => { | |
| scope.postMessage({ type: 'fatal', message: error.message }); | |
| throw error; | |
| }); | |
| break; | |
| case 'cancel': | |
| latest = msg.id; | |
| break; | |
| case 'speak': | |
| void speak(msg); | |
| break; | |
| default: { | |
| const never: never = msg; | |
| throw new Error(`unknown message ${JSON.stringify(never)}`); | |
| } | |
| } | |
| }; | |