chenbhao commited on
Commit
b925a97
·
1 Parent(s): 103a8ed

feat: VAD Noise Suppression

Browse files

代码正确。运行现有测试确保没有破坏。

● Bash(npx vitest run src/friend/__tests__/ 2>&1 | tail -30)
⎿   FAIL src/friend/__tests__/e2e-http-flow.test.ts [ src/friend/__tests__/e2e-http-flow.test.ts ]
Error: Cannot find package 'bun:test' imported from /home/yuki/Code/Agent/VersperClaw/src/friend/__tests__/e2e-http-flow.test.ts
❯ src/friend/__tests__/e2e-http-flow.test.ts:10:1
… +26 lines (ctrl+o to expand)
⎿  (timeout 30s)

● 测试使用 bun,需要 bun test。

● Bash(bun test src/friend/__tests__/ 2>&1 | tail -30)
⎿  bun test v1.3.14-canary.1 (0d9b296a)

src/friend/__tests__/e2e-http-flow.test.ts:
… +15 lines (ctrl+o to expand)

● 4 测试全部通过,15 个 expect() 均无失败。

修改总结

vad-service.ts — 核心 VAD 检测逻辑

阈值调整(减少假阳性):

┌─────────────────────────┬───────────┬────────────┬─────────────────────────────────────┐
│ 参数 │ 旧值 │ 新值 │ 作用 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ positiveSpeechThreshold │ 0.50 │ 0.75 │ 需要 Silero 75% 置信才判定为语音 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ negativeSpeechThreshold │ 0.35 │ 0.50 │ 降到 50% 以下才算静音(hysteresis) │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ redemptionFrames │ 8 (256ms) │ 20 (640ms) │ 更长的消音期,避免状态快速切换 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ minSpeechFrames │ 3 (96ms) │ 6 (192ms) │ 最低语音帧数提高 │
├─────────────────────────┼───────────┼────────────┼─────────────────────────────────────┤
│ preSpeechTriggerFrames │ — │ 10 (320ms) │ 新增:需要连续 10 帧语音才触发 │
└─────────────────────────┴───────────┴────────────┴─────────────────────────────────────┘

新增 RMS 能量预过滤(rmsThreshold: 0.004):
- 麦克风碰撞、摩擦、桌面震动等非语音低频噪声,RMS 低于此值时直接跳过 Silero 推理
- 设置为 prob=0,不会误导状态机

新增前置语音帧检测阶段(pre-speech phase):
- else if (isSpeech) 不再立即进入 speaking 状态
- 先进入 pre-speech 阶段累计 preSpeechCount
- 直到连续 preSpeechTriggerFrames 帧都超过阈值才正式触发 onSpeechStart
- 短促爆音(<320ms)会被完全忽略

状态机流程:
非语音帧 → preSpeechCount=0
语音帧1 → preSpeechCount=1, 等待
语音帧2 → preSpeechCount=2, 等待
...
语音帧10 → preSpeechCount=10 ≥ 10 → speaking=true, onSpeechStart()
语音帧11+ → 正常说话状态
静音帧 → redemptionCounter++ → 达到 20 → onSpeechEnd()

FriendService.ts — SileroVad 构造参数

向 SileroVad 构造函数传入显式的严格参数,覆盖默认值,确保生产环境中使用保守策略。

测试

全部 4 个已有测试通过(15 个 expect 断言),无回归。

src/friend/FriendService.ts CHANGED
@@ -116,6 +116,14 @@ class FriendService {
116
  console.error('[FriendService] VAD segment flush error:', e),
117
  );
118
  },
 
 
 
 
 
 
 
 
119
  });
120
  vad.init().then(() => {
121
  this.vadInstance = vad;
 
116
  console.error('[FriendService] VAD segment flush error:', e),
117
  );
118
  },
119
+ }, {
120
+ // Stricter thresholds to reduce false positives from non-speech noise
121
+ positiveSpeechThreshold: 0.75, // need 75% confidence
122
+ negativeSpeechThreshold: 0.50, // must drop below 50% to stop
123
+ preSpeechTriggerFrames: 10, // require ~320ms sustained speech to trigger
124
+ minSpeechFrames: 6, // ~192ms minimum confirmed speech
125
+ redemptionFrames: 20, // ~640ms silence before segment ends
126
+ rmsThreshold: 0.004, // -48dBFS noise floor
127
  });
128
  vad.init().then(() => {
129
  this.vadInstance = vad;
src/friend/voice/vad-service.ts CHANGED
@@ -24,18 +24,22 @@ export interface VadCallbacks {
24
  }
25
 
26
  export interface VadOptions {
27
- /** Threshold above which a frame is considered speech (0-1). Default: 0.5 */
28
  positiveSpeechThreshold?: number;
29
- /** Threshold below which a frame is considered silence (0-1). Default: 0.35 */
30
  negativeSpeechThreshold?: number;
31
- /** Consecutive silence frames before onSpeechEnd fires. Default: 8 (~256ms) */
32
  redemptionFrames?: number;
33
- /** Minimum speech frames to avoid misfire. Default: 3 (~96ms) */
34
  minSpeechFrames?: number;
35
  /** Frames of pre-speech audio to include in onSpeechEnd segment. Default: 10 */
36
  preSpeechPadFrames?: number;
37
  /** Sample rate of input audio (must be 16000). Default: 16000 */
38
  sampleRate?: number;
 
 
 
 
39
  }
40
 
41
  export class SileroVad {
@@ -57,6 +61,8 @@ export class SileroVad {
57
  private speaking = false;
58
  private redemptionCounter = 0;
59
  private speechFrameCount = 0;
 
 
60
  private frameHistory: Array<{ frame: Float32Array; isSpeech: boolean }> = [];
61
 
62
  constructor(callbacks: VadCallbacks, opts?: VadOptions) {
@@ -67,12 +73,14 @@ export class SileroVad {
67
  onFrameProcessed: callbacks.onFrameProcessed ?? (() => {}),
68
  };
69
  this.opts = {
70
- positiveSpeechThreshold: opts?.positiveSpeechThreshold ?? 0.5,
71
- negativeSpeechThreshold: opts?.negativeSpeechThreshold ?? 0.35,
72
- redemptionFrames: opts?.redemptionFrames ?? 8,
73
- minSpeechFrames: opts?.minSpeechFrames ?? 3,
74
  preSpeechPadFrames: opts?.preSpeechPadFrames ?? 10,
75
  sampleRate: opts?.sampleRate ?? 16000,
 
 
76
  };
77
  }
78
 
@@ -153,6 +161,7 @@ export class SileroVad {
153
  this.speaking = false;
154
  this.redemptionCounter = 0;
155
  this.speechFrameCount = 0;
 
156
  this.stateH = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
157
  this.stateC = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
158
  }
@@ -173,49 +182,71 @@ export class SileroVad {
173
  private async processFrame(frame: Float32Array): Promise<void> {
174
  if (!this.session || !this.stateH || !this.stateC || !this.sr) return;
175
 
176
- try {
177
- const result = await this.session.run({
178
- input: new ort.Tensor('float32', frame, [1, this.frameSize]),
179
- sr: this.sr,
180
- h: this.stateH,
181
- c: this.stateC,
182
- });
183
-
184
- // Update LSTM state for next frame
185
- this.stateH = result.hn as ort.Tensor<onnxruntime.TensorType>;
186
- this.stateC = result.cn as ort.Tensor<onnxruntime.TensorType>;
187
-
188
- const prob = (result.output as ort.Tensor<onnxruntime.TensorType>).data[0] as number;
189
- const isSpeech = prob >= this.opts.positiveSpeechThreshold;
190
- const isSilence = prob < this.opts.negativeSpeechThreshold;
191
-
192
- this.callbacks.onFrameProcessed(prob, isSpeech);
193
-
194
- // ── State machine ──────────────────────────────────────────────
195
- if (this.speaking) {
196
- // Currently in a speech segment
197
- if (isSilence) {
198
- this.redemptionCounter++;
199
- if (this.redemptionCounter >= this.opts.redemptionFrames) {
200
- this.endSpeech();
201
- }
202
- } else {
203
- this.redemptionCounter = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
204
  }
205
- this.speechFrameCount++;
206
- this.frameHistory.push({ frame: frame.slice(), isSpeech });
207
- } else if (isSpeech) {
208
- // Transition: silence → speech
 
 
 
 
 
 
 
209
  this.speaking = true;
210
  this.redemptionCounter = 0;
211
- this.speechFrameCount = 1;
212
-
213
- // Include pre-padding frames (audio just before speech start)
214
  this.frameHistory.push({ frame: frame.slice(), isSpeech: true });
215
  this.callbacks.onSpeechStart();
216
  }
217
- } catch (err) {
218
- console.error('[SileroVad] frame inference error:', err);
 
219
  }
220
  }
221
 
 
24
  }
25
 
26
  export interface VadOptions {
27
+ /** Threshold above which a frame is considered speech (0-1). Default: 0.75 */
28
  positiveSpeechThreshold?: number;
29
+ /** Threshold below which a frame is considered silence (0-1). Default: 0.50 */
30
  negativeSpeechThreshold?: number;
31
+ /** Consecutive silence frames before onSpeechEnd fires. Default: 20 (~640ms) */
32
  redemptionFrames?: number;
33
+ /** Minimum confirmed speech frames to avoid misfire. Default: 6 (~192ms) */
34
  minSpeechFrames?: number;
35
  /** Frames of pre-speech audio to include in onSpeechEnd segment. Default: 10 */
36
  preSpeechPadFrames?: number;
37
  /** Sample rate of input audio (must be 16000). Default: 16000 */
38
  sampleRate?: number;
39
+ /** RMS energy threshold (0-1). Frames below this are treated as silence without inference. Default: 0.004 (~-48dBFS) */
40
+ rmsThreshold?: number;
41
+ /** Consecutive speech frames required to trigger speech start. Default: 10 (~320ms) — filters short noise bursts */
42
+ preSpeechTriggerFrames?: number;
43
  }
44
 
45
  export class SileroVad {
 
61
  private speaking = false;
62
  private redemptionCounter = 0;
63
  private speechFrameCount = 0;
64
+ /** Consecutive speech frame count in pre-speech phase (fires speech on threshold) */
65
+ private preSpeechCount = 0;
66
  private frameHistory: Array<{ frame: Float32Array; isSpeech: boolean }> = [];
67
 
68
  constructor(callbacks: VadCallbacks, opts?: VadOptions) {
 
73
  onFrameProcessed: callbacks.onFrameProcessed ?? (() => {}),
74
  };
75
  this.opts = {
76
+ positiveSpeechThreshold: opts?.positiveSpeechThreshold ?? 0.75,
77
+ negativeSpeechThreshold: opts?.negativeSpeechThreshold ?? 0.50,
78
+ redemptionFrames: opts?.redemptionFrames ?? 20,
79
+ minSpeechFrames: opts?.minSpeechFrames ?? 6,
80
  preSpeechPadFrames: opts?.preSpeechPadFrames ?? 10,
81
  sampleRate: opts?.sampleRate ?? 16000,
82
+ rmsThreshold: opts?.rmsThreshold ?? 0.004,
83
+ preSpeechTriggerFrames: opts?.preSpeechTriggerFrames ?? 10,
84
  };
85
  }
86
 
 
161
  this.speaking = false;
162
  this.redemptionCounter = 0;
163
  this.speechFrameCount = 0;
164
+ this.preSpeechCount = 0;
165
  this.stateH = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
166
  this.stateC = new ort.Tensor('float32', new Float32Array(2 * 64), [2, 1, 64]);
167
  }
 
182
  private async processFrame(frame: Float32Array): Promise<void> {
183
  if (!this.session || !this.stateH || !this.stateC || !this.sr) return;
184
 
185
+ // ── 1. Energy pre-filter: compute RMS — skip Silero for low-energy noise ──
186
+ let sumSq = 0;
187
+ for (let i = 0; i < frame.length; i++) {
188
+ sumSq += frame[i] * frame[i];
189
+ }
190
+ const rms = Math.sqrt(sumSq / frame.length);
191
+
192
+ let prob: number;
193
+ if (rms < this.opts.rmsThreshold) {
194
+ prob = 0; // Below noise floor — mechanical noise, mic bump, room silence
195
+ } else {
196
+ // ── 2. Silero inference for speech probability ──
197
+ try {
198
+ const result = await this.session.run({
199
+ input: new ort.Tensor('float32', frame, [1, this.frameSize]),
200
+ sr: this.sr,
201
+ h: this.stateH,
202
+ c: this.stateC,
203
+ });
204
+
205
+ // Update LSTM state for next frame
206
+ this.stateH = result.hn as ort.Tensor<onnxruntime.TensorType>;
207
+ this.stateC = result.cn as ort.Tensor<onnxruntime.TensorType>;
208
+
209
+ prob = (result.output as ort.Tensor<onnxruntime.TensorType>).data[0] as number;
210
+ } catch (err) {
211
+ console.error('[SileroVad] frame inference error:', err);
212
+ return;
213
+ }
214
+ }
215
+
216
+ const isSpeech = prob >= this.opts.positiveSpeechThreshold;
217
+ const isSilence = prob < this.opts.negativeSpeechThreshold;
218
+
219
+ this.callbacks.onFrameProcessed(prob, isSpeech);
220
+
221
+ // ── 3. State machine ────────────────────────────────────────────
222
+ if (this.speaking) {
223
+ // In a confirmed speech segment
224
+ if (isSilence) {
225
+ this.redemptionCounter++;
226
+ if (this.redemptionCounter >= this.opts.redemptionFrames) {
227
+ this.endSpeech();
228
  }
229
+ } else {
230
+ this.redemptionCounter = 0;
231
+ }
232
+ this.speechFrameCount++;
233
+ this.frameHistory.push({ frame: frame.slice(), isSpeech });
234
+ } else if (isSpeech) {
235
+ // Pre-speech phase: require consecutive speech frames to trigger
236
+ this.preSpeechCount++;
237
+
238
+ if (this.preSpeechCount >= this.opts.preSpeechTriggerFrames) {
239
+ // Transition: silence → confirmed speech (sustained above threshold)
240
  this.speaking = true;
241
  this.redemptionCounter = 0;
242
+ this.speechFrameCount = this.preSpeechCount;
243
+ this.preSpeechCount = 0;
 
244
  this.frameHistory.push({ frame: frame.slice(), isSpeech: true });
245
  this.callbacks.onSpeechStart();
246
  }
247
+ } else {
248
+ // Not speech discard any accumulated pre-speech frames
249
+ this.preSpeechCount = 0;
250
  }
251
  }
252