feat: VAD Noise Suppression
Browse files代码正确。运行现有测试确保没有破坏。
● Bash(npx vitest run src/friend/__tests__/ 2>&1 | tail -30)
⎿ FAIL src/friend/__tests__/e2e-http-flow.test.ts [ src/friend/__tests__/e2e-http-flow.test.ts ]
Error: Cannot find package 'bun:test' imported from /home/yuki/Code/Agent/VersperClaw/src/friend/__tests__/e2e-http-flow.test.ts
❯ src/friend/__tests__/e2e-http-flow.test.ts:10:1
… +26 lines (ctrl+o to expand)
⎿ (timeout 30s)
● 测试使用 bun,需要 bun test。
● Bash(bun test src/friend/__tests__/ 2>&1 | tail -30)
⎿ bun test v1.3.14-canary.1 (0d9b296a)
src/friend/__tests__/e2e-http-flow.test.ts:
… +15 lines (ctrl+o to expand)
● 4 测试全部通过,15 个 expect() 均无失败。
修改总结
vad-service.ts — 核心 VAD 检测逻辑
阈值调整(减少假阳性):
┌─────────────────────────┬───────────┬────────────┬─────────────────────────────────────┐
│ 参数 │ 旧值 │ 新值 │ 作用 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ positiveSpeechThreshold │ 0.50 │ 0.75 │ 需要 Silero 75% 置信才判定为语音 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ negativeSpeechThreshold │ 0.35 │ 0.50 │ 降到 50% 以下才算静音(hysteresis) │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ redemptionFrames │ 8 (256ms) │ 20 (640ms) │ 更长的消音期,避免状态快速切换 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ minSpeechFrames │ 3 (96ms) │ 6 (192ms) │ 最低语音帧数提高 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ preSpeechTriggerFrames │ — │ 10 (320ms) │ 新增:需要连续 10 帧语音才触发 │
└─────────────────────────┴───────────┴────────────┴─────────────────────────────────────┘
新增 RMS 能量预过滤(rmsThreshold: 0.004):
- 麦克风碰撞、摩擦、桌面震动等非语音低频噪声,RMS 低于此值时直接跳过 Silero 推理
- 设置为 prob=0,不会误导状态机
新增前置语音帧检测阶段(pre-speech phase):
- else if (isSpeech) 不再立即进入 speaking 状态
- 先进入 pre-speech 阶段累计 preSpeechCount
- 直到连续 preSpeechTriggerFrames 帧都超过阈值才正式触发 onSpeechStart
- 短促爆音(<320ms)会被完全忽略
状态机流程:
非语音帧 → preSpeechCount=0
语音帧1 → preSpeechCount=1, 等待
语音帧2 → preSpeechCount=2, 等待
...
语音帧10 → preSpeechCount=10 ≥ 10 → speaking=true, onSpeechStart()
语音帧11+ → 正常说话状态
静音帧 → redemptionCounter++ → 达到 20 → onSpeechEnd()
FriendService.ts — SileroVad 构造参数
向 SileroVad 构造函数传入显式的严格参数,覆盖默认值,确保生产环境中使用保守策略。
测试
全部 4 个已有测试通过(15 个 expect 断言),无回归。
- src/friend/FriendService.ts +8 -0
- src/friend/voice/vad-service.ts +76 -45
|
@@ -116,6 +116,14 @@ class FriendService {
|
|
| 116 |
console.error('[FriendService] VAD segment flush error:', e),
|
| 117 |
);
|
| 118 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
});
|
| 120 |
vad.init().then(() => {
|
| 121 |
this.vadInstance = vad;
|
|
|
|
| 116 |
console.error('[FriendService] VAD segment flush error:', e),
|
| 117 |
);
|
| 118 |
},
|
| 119 |
+
}, {
|
| 120 |
+
// Stricter thresholds to reduce false positives from non-speech noise
|
| 121 |
+
positiveSpeechThreshold: 0.75, // need 75% confidence
|
| 122 |
+
negativeSpeechThreshold: 0.50, // must drop below 50% to stop
|
| 123 |
+
preSpeechTriggerFrames: 10, // require ~320ms sustained speech to trigger
|
| 124 |
+
minSpeechFrames: 6, // ~192ms minimum confirmed speech
|
| 125 |
+
redemptionFrames: 20, // ~640ms silence before segment ends
|
| 126 |
+
rmsThreshold: 0.004, // -48dBFS noise floor
|
| 127 |
});
|
| 128 |
vad.init().then(() => {
|
| 129 |
this.vadInstance = vad;
|
|
@@ -24,18 +24,22 @@ export interface VadCallbacks {
|
|
| 24 |
}
|
| 25 |
|
| 26 |
export interface VadOptions {
|
| 27 |
-
/** Threshold above which a frame is considered speech (0-1). Default: 0.
|
| 28 |
positiveSpeechThreshold?: number;
|
| 29 |
-
/** Threshold below which a frame is considered silence (0-1). Default: 0.
|
| 30 |
negativeSpeechThreshold?: number;
|
| 31 |
-
/** Consecutive silence frames before onSpeechEnd fires. Default:
|
| 32 |
redemptionFrames?: number;
|
| 33 |
-
/** Minimum speech frames to avoid misfire. Default:
|
| 34 |
minSpeechFrames?: number;
|
| 35 |
/** Frames of pre-speech audio to include in onSpeechEnd segment. Default: 10 */
|
| 36 |
preSpeechPadFrames?: number;
|
| 37 |
/** Sample rate of input audio (must be 16000). Default: 16000 */
|
| 38 |
sampleRate?: number;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
}
|
| 40 |
|
| 41 |
export class SileroVad {
|
|
@@ -57,6 +61,8 @@ export class SileroVad {
|
|
| 57 |
private speaking = false;
|
| 58 |
private redemptionCounter = 0;
|
| 59 |
private speechFrameCount = 0;
|
|
|
|
|
|
|
| 60 |
private frameHistory: Array<{ frame: Float32Array; isSpeech: boolean }> = [];
|
| 61 |
|
| 62 |
constructor(callbacks: VadCallbacks, opts?: VadOptions) {
|
|
@@ -67,12 +73,14 @@ export class SileroVad {
|
|
| 67 |
onFrameProcessed: callbacks.onFrameProcessed ?? (() => {}),
|
| 68 |
};
|
| 69 |
this.opts = {
|
| 70 |
-
positiveSpeechThreshold: opts?.positiveSpeechThreshold ?? 0.
|
| 71 |
-
negativeSpeechThreshold: opts?.negativeSpeechThreshold ?? 0.
|
| 72 |
-
redemptionFrames: opts?.redemptionFrames ??
|
| 73 |
-
minSpeechFrames: opts?.minSpeechFrames ??
|
| 74 |
preSpeechPadFrames: opts?.preSpeechPadFrames ?? 10,
|
| 75 |
sampleRate: opts?.sampleRate ?? 16000,
|
|
|
|
|
|
|
| 76 |
};
|
| 77 |
}
|
| 78 |
|
|
@@ -153,6 +161,7 @@ export class SileroVad {
|
|
| 153 |
this.speaking = false;
|
| 154 |
this.redemptionCounter = 0;
|
| 155 |
this.speechFrameCount = 0;
|
|
|
|
| 156 |
this.stateH = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
|
| 157 |
this.stateC = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
|
| 158 |
}
|
|
@@ -173,49 +182,71 @@ export class SileroVad {
|
|
| 173 |
private async processFrame(frame: Float32Array): Promise<void> {
|
| 174 |
if (!this.session || !this.stateH || !this.stateC || !this.sr) return;
|
| 175 |
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
//
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
}
|
| 205 |
-
|
| 206 |
-
this.
|
| 207 |
-
}
|
| 208 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
this.speaking = true;
|
| 210 |
this.redemptionCounter = 0;
|
| 211 |
-
this.speechFrameCount =
|
| 212 |
-
|
| 213 |
-
// Include pre-padding frames (audio just before speech start)
|
| 214 |
this.frameHistory.push({ frame: frame.slice(), isSpeech: true });
|
| 215 |
this.callbacks.onSpeechStart();
|
| 216 |
}
|
| 217 |
-
}
|
| 218 |
-
|
|
|
|
| 219 |
}
|
| 220 |
}
|
| 221 |
|
|
|
|
| 24 |
}
|
| 25 |
|
| 26 |
export interface VadOptions {
|
| 27 |
+
/** Threshold above which a frame is considered speech (0-1). Default: 0.75 */
|
| 28 |
positiveSpeechThreshold?: number;
|
| 29 |
+
/** Threshold below which a frame is considered silence (0-1). Default: 0.50 */
|
| 30 |
negativeSpeechThreshold?: number;
|
| 31 |
+
/** Consecutive silence frames before onSpeechEnd fires. Default: 20 (~640ms) */
|
| 32 |
redemptionFrames?: number;
|
| 33 |
+
/** Minimum confirmed speech frames to avoid misfire. Default: 6 (~192ms) */
|
| 34 |
minSpeechFrames?: number;
|
| 35 |
/** Frames of pre-speech audio to include in onSpeechEnd segment. Default: 10 */
|
| 36 |
preSpeechPadFrames?: number;
|
| 37 |
/** Sample rate of input audio (must be 16000). Default: 16000 */
|
| 38 |
sampleRate?: number;
|
| 39 |
+
/** RMS energy threshold (0-1). Frames below this are treated as silence without inference. Default: 0.004 (~-48dBFS) */
|
| 40 |
+
rmsThreshold?: number;
|
| 41 |
+
/** Consecutive speech frames required to trigger speech start. Default: 10 (~320ms) — filters short noise bursts */
|
| 42 |
+
preSpeechTriggerFrames?: number;
|
| 43 |
}
|
| 44 |
|
| 45 |
export class SileroVad {
|
|
|
|
| 61 |
private speaking = false;
|
| 62 |
private redemptionCounter = 0;
|
| 63 |
private speechFrameCount = 0;
|
| 64 |
+
/** Consecutive speech frame count in pre-speech phase (fires speech on threshold) */
|
| 65 |
+
private preSpeechCount = 0;
|
| 66 |
private frameHistory: Array<{ frame: Float32Array; isSpeech: boolean }> = [];
|
| 67 |
|
| 68 |
constructor(callbacks: VadCallbacks, opts?: VadOptions) {
|
|
|
|
| 73 |
onFrameProcessed: callbacks.onFrameProcessed ?? (() => {}),
|
| 74 |
};
|
| 75 |
this.opts = {
|
| 76 |
+
positiveSpeechThreshold: opts?.positiveSpeechThreshold ?? 0.75,
|
| 77 |
+
negativeSpeechThreshold: opts?.negativeSpeechThreshold ?? 0.50,
|
| 78 |
+
redemptionFrames: opts?.redemptionFrames ?? 20,
|
| 79 |
+
minSpeechFrames: opts?.minSpeechFrames ?? 6,
|
| 80 |
preSpeechPadFrames: opts?.preSpeechPadFrames ?? 10,
|
| 81 |
sampleRate: opts?.sampleRate ?? 16000,
|
| 82 |
+
rmsThreshold: opts?.rmsThreshold ?? 0.004,
|
| 83 |
+
preSpeechTriggerFrames: opts?.preSpeechTriggerFrames ?? 10,
|
| 84 |
};
|
| 85 |
}
|
| 86 |
|
|
|
|
| 161 |
this.speaking = false;
|
| 162 |
this.redemptionCounter = 0;
|
| 163 |
this.speechFrameCount = 0;
|
| 164 |
+
this.preSpeechCount = 0;
|
| 165 |
this.stateH = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
|
| 166 |
this.stateC = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
|
| 167 |
}
|
|
|
|
| 182 |
private async processFrame(frame: Float32Array): Promise<void> {
|
| 183 |
if (!this.session || !this.stateH || !this.stateC || !this.sr) return;
|
| 184 |
|
| 185 |
+
// ── 1. Energy pre-filter: compute RMS — skip Silero for low-energy noise ──
|
| 186 |
+
let sumSq = 0;
|
| 187 |
+
for (let i = 0; i < frame.length; i++) {
|
| 188 |
+
sumSq += frame[i] * frame[i];
|
| 189 |
+
}
|
| 190 |
+
const rms = Math.sqrt(sumSq / frame.length);
|
| 191 |
+
|
| 192 |
+
let prob: number;
|
| 193 |
+
if (rms < this.opts.rmsThreshold) {
|
| 194 |
+
prob = 0; // Below noise floor — mechanical noise, mic bump, room silence
|
| 195 |
+
} else {
|
| 196 |
+
// ── 2. Silero inference for speech probability ──
|
| 197 |
+
try {
|
| 198 |
+
const result = await this.session.run({
|
| 199 |
+
input: new ort.Tensor('float32', frame, [1, this.frameSize]),
|
| 200 |
+
sr: this.sr,
|
| 201 |
+
h: this.stateH,
|
| 202 |
+
c: this.stateC,
|
| 203 |
+
});
|
| 204 |
+
|
| 205 |
+
// Update LSTM state for next frame
|
| 206 |
+
this.stateH = result.hn as ort.Tensor<onnxruntime.TensorType>;
|
| 207 |
+
this.stateC = result.cn as ort.Tensor<onnxruntime.TensorType>;
|
| 208 |
+
|
| 209 |
+
prob = (result.output as ort.Tensor<onnxruntime.TensorType>).data[0] as number;
|
| 210 |
+
} catch (err) {
|
| 211 |
+
console.error('[SileroVad] frame inference error:', err);
|
| 212 |
+
return;
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
const isSpeech = prob >= this.opts.positiveSpeechThreshold;
|
| 217 |
+
const isSilence = prob < this.opts.negativeSpeechThreshold;
|
| 218 |
+
|
| 219 |
+
this.callbacks.onFrameProcessed(prob, isSpeech);
|
| 220 |
+
|
| 221 |
+
// ── 3. State machine ────────────────────────────────────────────
|
| 222 |
+
if (this.speaking) {
|
| 223 |
+
// In a confirmed speech segment
|
| 224 |
+
if (isSilence) {
|
| 225 |
+
this.redemptionCounter++;
|
| 226 |
+
if (this.redemptionCounter >= this.opts.redemptionFrames) {
|
| 227 |
+
this.endSpeech();
|
| 228 |
}
|
| 229 |
+
} else {
|
| 230 |
+
this.redemptionCounter = 0;
|
| 231 |
+
}
|
| 232 |
+
this.speechFrameCount++;
|
| 233 |
+
this.frameHistory.push({ frame: frame.slice(), isSpeech });
|
| 234 |
+
} else if (isSpeech) {
|
| 235 |
+
// Pre-speech phase: require consecutive speech frames to trigger
|
| 236 |
+
this.preSpeechCount++;
|
| 237 |
+
|
| 238 |
+
if (this.preSpeechCount >= this.opts.preSpeechTriggerFrames) {
|
| 239 |
+
// Transition: silence → confirmed speech (sustained above threshold)
|
| 240 |
this.speaking = true;
|
| 241 |
this.redemptionCounter = 0;
|
| 242 |
+
this.speechFrameCount = this.preSpeechCount;
|
| 243 |
+
this.preSpeechCount = 0;
|
|
|
|
| 244 |
this.frameHistory.push({ frame: frame.slice(), isSpeech: true });
|
| 245 |
this.callbacks.onSpeechStart();
|
| 246 |
}
|
| 247 |
+
} else {
|
| 248 |
+
// Not speech — discard any accumulated pre-speech frames
|
| 249 |
+
this.preSpeechCount = 0;
|
| 250 |
}
|
| 251 |
}
|
| 252 |
|