Hugging Face's logo

Spaces:
taqarub
/
node-api
Sleeping

App Files Community
node-api / server /pcm-segmenter.ts
taqarub's picture
taqarub
update persistent PCM voice bridge
79f41f5
Raw History Blame Contribute Delete
12 kB
/**
* pcm-segmenter.ts — Continuous PCM16 utterance segmenter for streaming STT.
*
* Turns an endless stream of raw PCM16LE mono audio into discrete utterance
* segments suitable for a request/response STT API (Whisper), without losing
* speech at segment boundaries:
*
* - PRE-ROLL: audio captured just BEFORE speech onset is prepended to the
* utterance so soft word onsets are not clipped.
* - HANGOVER: an utterance only ends after a sustained silence, so trailing
* syllables and short pauses inside a sentence are kept.
* - ADAPTIVE VAD: the speech threshold tracks the noise floor, so it works
* for both close-mic speech and external audio (TV, videos, news).
* - FORCED CUTS: continuous speech longer than `maxUtteranceMs` is split at
* the QUIETEST recent window (least likely to be mid-word) instead of an
* arbitrary time boundary.
*
* The class is pure (no I/O): callers feed PCM buffers in and receive
* segment buffers via the `onSegment` callback.
*/
// ─── WAV encoding ────────────────────────────────────────────────────────────
/** Wrap raw PCM16LE mono samples in a 44-byte WAV header. */
export function pcm16ToWav(pcm: Buffer, sampleRate: number): Buffer {
const header = Buffer.alloc(44);
const byteRate = sampleRate * 2; // mono, 16-bit
header.write("RIFF", 0);
header.writeUInt32LE(36 + pcm.length, 4);
header.write("WAVE", 8);
header.write("fmt ", 12);
header.writeUInt32LE(16, 16); // fmt chunk size
header.writeUInt16LE(1, 20); // PCM format
header.writeUInt16LE(1, 22); // mono
header.writeUInt32LE(sampleRate, 24);
header.writeUInt32LE(byteRate, 28);
header.writeUInt16LE(2, 32); // block align
header.writeUInt16LE(16, 34); // bits per sample
header.write("data", 36);
header.writeUInt32LE(pcm.length, 40);
return Buffer.concat([header, pcm]);
}
// ─── Segmenter ───────────────────────────────────────────────────────────────
export interface PcmSegmenterOptions {
/** Sample rate of the incoming PCM16LE mono stream. Default: 16000 */
sampleRate?: number;
/** Audio kept from before speech onset. Default: 320ms */
preRollMs?: number;
/** Sustained silence that ends an utterance. Default: 900ms */
hangoverMs?: number;
/** Utterances with less speech than this are discarded as noise. Default: 180ms */
minSpeechMs?: number;
/** Continuous speech is force-cut after this long. Default: 7000ms */
maxUtteranceMs?: number;
/** Absolute minimum level (dBFS) treated as speech. Default: -48 */
speechFloorDb?: number;
/** Speech must exceed the noise floor by this margin (dB). Default: 9 */
noiseMarginDb?: number;
}
export interface SegmentInfo {
/** True when the segment was cut mid-speech (maxUtteranceMs reached). */
forced: boolean;
/** Audio duration of the emitted segment in milliseconds. */
durationMs: number;
/** Milliseconds of the segment classified as speech. */
speechMs: number;
}
interface Window {
buf: Buffer;
db: number;
speech: boolean;
}
/** VAD analysis window; 20ms is fine-grained enough for word boundaries. */
const WINDOW_MS = 20;
/** Trailing silence kept on an emitted segment (helps STT decode the tail). */
const TAIL_SILENCE_KEEP_MS = 240;
/** Forced cuts search for the quietest window within this recent span. */
const FORCED_CUT_SEARCH_MS = 2000;
/** Forced cuts never split inside the newest audio (still mid-word). */
const FORCED_CUT_GUARD_MS = 200;
export class PcmSegmenter {
private readonly sampleRate: number;
private readonly preRollMs: number;
private readonly hangoverMs: number;
private readonly minSpeechMs: number;
private readonly maxUtteranceMs: number;
private readonly speechFloorDb: number;
private readonly noiseMarginDb: number;
private readonly windowBytes: number;
/** Carry-over for incoming buffers that don't align to whole windows. */
private remainder: Buffer = Buffer.alloc(0);
/** Rolling buffer of recent windows while idle (pre-roll source). */
private preRoll: Window[] = [];
/** Windows of the in-progress utterance (empty while idle). */
private active: Window[] = [];
private activeSpeechMs = 0;
private silenceRunMs = 0;
/** EMA noise floor in dBFS; rises slowly, falls quickly. */
private noiseFloorDb = -65;
constructor(
options: PcmSegmenterOptions,
private readonly onSegment: (pcm: Buffer, info: SegmentInfo) => void,
) {
this.sampleRate = options.sampleRate ?? 16000;
this.preRollMs = options.preRollMs ?? 320;
this.hangoverMs = options.hangoverMs ?? 900;
this.minSpeechMs = options.minSpeechMs ?? 180;
this.maxUtteranceMs = options.maxUtteranceMs ?? 7000;
this.speechFloorDb = options.speechFloorDb ?? -48;
this.noiseMarginDb = options.noiseMarginDb ?? 9;
this.windowBytes = Math.round((this.sampleRate * WINDOW_MS) / 1000) * 2;
}
/** Feed the next PCM16LE buffer from the stream. */
feed(pcm: Buffer): void {
const data = this.remainder.length
? Buffer.concat([this.remainder, pcm])
: pcm;
const wholeWindows = Math.floor(data.length / this.windowBytes);
const usedBytes = wholeWindows * this.windowBytes;
this.remainder = data.subarray(usedBytes);
for (let i = 0; i < wholeWindows; i++) {
const buf = data.subarray(i * this.windowBytes, (i + 1) * this.windowBytes);
this.processWindow(buf);
}
}
/**
* Finalize the in-progress utterance immediately (stream stopped).
* Emits the tail segment if it contains enough speech.
*/
flush(): void {
if (this.remainder.length >= 4) {
// Analyse the sub-window remainder too — a stream can stop mid-window.
this.processWindow(this.remainder);
}
this.remainder = Buffer.alloc(0);
this.endUtterance(false);
}
/** Drop all buffered state (new session). Keeps the learned noise floor. */
reset(): void {
this.remainder = Buffer.alloc(0);
this.preRoll = [];
this.active = [];
this.activeSpeechMs = 0;
this.silenceRunMs = 0;
}
/**
* Snapshot of the in-progress utterance, for rolling partial transcription.
* Returns null while idle or when the utterance is still too short.
*/
getActiveSnapshot(): { pcm: Buffer; durationMs: number; speechMs: number } | null {
if (!this.active.length || this.activeSpeechMs < this.minSpeechMs) return null;
const pcm = Buffer.concat(this.active.map((w) => w.buf));
return {
pcm,
durationMs: this.msOf(pcm),
speechMs: this.activeSpeechMs,
};
}
// ── Internals ────────────────────────────────────────────────────────────
private msOf(buf: Buffer): number {
return (buf.length / 2 / this.sampleRate) * 1000;
}
private speechThresholdDb(): number {
return Math.max(this.speechFloorDb, this.noiseFloorDb + this.noiseMarginDb);
}
private processWindow(buf: Buffer): void {
const db = rmsDb(buf);
const speech = db >= this.speechThresholdDb();
const win: Window = { buf, db, speech };
const winMs = this.msOf(buf);
// Track the noise floor from non-speech windows only, capped so loud
// continuous audio can never drag the floor up into speech territory.
if (!speech) {
const alpha = db < this.noiseFloorDb ? 0.2 : 0.02;
this.noiseFloorDb = Math.min(
-35,
this.noiseFloorDb + alpha * (db - this.noiseFloorDb),
);
}
if (!this.active.length) {
if (speech) {
// Speech onset: start the utterance with the pre-roll prepended.
this.active = [...this.preRoll, win];
this.preRoll = [];
this.activeSpeechMs = winMs;
this.silenceRunMs = 0;
} else {
this.preRoll.push(win);
let total = this.preRoll.reduce((s, w) => s + this.msOf(w.buf), 0);
while (this.preRoll.length > 1 && total > this.preRollMs) {
total -= this.msOf(this.preRoll[0].buf);
this.preRoll.shift();
}
}
return;
}
// Utterance in progress: keep everything (speech AND intra-speech silence).
this.active.push(win);
if (speech) {
this.activeSpeechMs += winMs;
this.silenceRunMs = 0;
} else {
this.silenceRunMs += winMs;
}
if (this.silenceRunMs >= this.hangoverMs) {
this.endUtterance(false);
return;
}
const activeMs = this.active.reduce((s, w) => s + this.msOf(w.buf), 0);
if (activeMs >= this.maxUtteranceMs) {
this.forceCut();
}
}
private endUtterance(forced: boolean): void {
if (!this.active.length) return;
let windows = this.active;
this.active = [];
const speechMs = this.activeSpeechMs;
this.activeSpeechMs = 0;
this.silenceRunMs = 0;
// Trim trailing silence beyond a short tail; the trimmed silence seeds
// the pre-roll so the next onset still has context.
let tailSilence = 0;
while (tailSilence < windows.length && !windows[windows.length - 1 - tailSilence].speech) {
tailSilence++;
}
const keepTail = Math.ceil(TAIL_SILENCE_KEEP_MS / WINDOW_MS);
if (tailSilence > keepTail) {
const cut = windows.length - (tailSilence - keepTail);
this.preRoll = windows.slice(cut);
windows = windows.slice(0, cut);
}
if (speechMs < this.minSpeechMs) {
// Noise blip — don't transcribe it, but keep its tail as pre-roll so a
// real onset right after still gets its leading audio.
this.preRoll = [...windows.slice(-Math.ceil(this.preRollMs / WINDOW_MS)), ...this.preRoll];
return;
}
const pcm = Buffer.concat(windows.map((w) => w.buf));
this.onSegment(pcm, {
forced,
durationMs: this.msOf(pcm),
speechMs,
});
}
/**
* Continuous speech reached maxUtteranceMs: split at the quietest window in
* the recent past (least likely to be mid-word). Audio after the cut point
* becomes the start of the next utterance — nothing is dropped.
*/
private forceCut(): void {
const searchWindows = Math.ceil(FORCED_CUT_SEARCH_MS / WINDOW_MS);
const guardWindows = Math.ceil(FORCED_CUT_GUARD_MS / WINDOW_MS);
const end = Math.max(1, this.active.length - guardWindows);
const start = Math.max(1, end - searchWindows);
let cut = end - 1;
for (let i = start; i < end; i++) {
if (this.active[i].db < this.active[cut].db) cut = i;
}
const emitted = this.active.slice(0, cut);
const carried = this.active.slice(cut);
const emittedSpeechMs = emitted.reduce(
(s, w) => s + (w.speech ? this.msOf(w.buf) : 0),
0,
);
this.active = carried;
this.activeSpeechMs = carried.reduce(
(s, w) => s + (w.speech ? this.msOf(w.buf) : 0),
0,
);
// silenceRunMs continues counting across the cut; recompute from the tail.
let silence = 0;
for (let i = carried.length - 1; i >= 0 && !carried[i].speech; i--) {
silence += this.msOf(carried[i].buf);
}
this.silenceRunMs = silence;
if (emittedSpeechMs < this.minSpeechMs || !emitted.length) return;
const pcm = Buffer.concat(emitted.map((w) => w.buf));
this.onSegment(pcm, {
forced: true,
durationMs: this.msOf(pcm),
speechMs: emittedSpeechMs,
});
}
}
/** RMS level of a PCM16LE buffer in dBFS. */
function rmsDb(buf: Buffer): number {
const samples = buf.length >> 1;
if (!samples) return -Infinity;
let sumSq = 0;
for (let i = 0; i < samples; i++) {
const s = buf.readInt16LE(i * 2) / 32768;
sumSq += s * s;
}
const rms = Math.sqrt(sumSq / samples);
return 20 * Math.log10(rms + 1e-10);
}