chenbhao commited on
Commit
11ee0ea
·
1 Parent(s): 0bed40c

fix: echo in VAD STT TTS

Browse files

回声问题修复

新增静音机制 — 在 AI 通过 TTS 回答期间,暂时阻断麦克风音频进入 STT 和 VAD:

1. muted 标志 + muteTimer — 控制音频采集的静音状态
2. muteForTts(text) — 根据回复文本长度估算 TTS 播放时长,自动管理静音期:
- 估算公式:max(3000ms, text.length × 100ms + 1500ms)
- 中文字符约 125ms/字,英文约 83ms/字符,+1s 生成/网络开销 + 0.5s 余量
- 最短 3 秒
3. clearMute() — 用户按 F2 结束通话时立即清除静音状态
4. arecord 回调中的 feedAudio(c) — 静音时直接 return,音频不进入 STT 也不进入 VAD

完整流程:
用户说话 → 音频 → STT → 转录 → AI → broadcastResponse()
↓ TTS生成成功 + 采集进行中
muteForTts(text)

静音期间所有音频丢弃
↓ (TTS播放完毕)
setTimeout → unmute

恢复音频采集

Files changed (1) hide show
  1. src/friend/FriendService.ts +63 -8
src/friend/FriendService.ts CHANGED
@@ -77,6 +77,10 @@ class FriendService {
77
  private _flushing = false;
78
  /** Silero VAD instance (real ML-based voice activity detection) */
79
  private vadInstance: SileroVad | null = null;
 
 
 
 
80
 
81
  // ── React sync external store interface ──────────────────────────────
82
 
@@ -415,6 +419,9 @@ class FriendService {
415
 
416
  this.capturing = false;
417
 
 
 
 
418
  // Reset VAD state silently (don't fire onSpeechEnd — we're finalizing below)
419
  if (this.vadInstance) {
420
  try {
@@ -496,6 +503,47 @@ class FriendService {
496
  return transcript;
497
  }
498
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
499
  // ── Response broadcast (called by useFriendBridge) ───────────────────
500
 
501
  /**
@@ -518,6 +566,11 @@ class FriendService {
518
  if (audioId) {
519
  const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
520
  broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
 
 
 
 
 
521
  }
522
  } catch (err) {
523
  console.warn('[FriendService] TTS generation failed:', err);
@@ -624,7 +677,14 @@ class FriendService {
624
  let verifyTimer: ReturnType<typeof setTimeout> | null = null;
625
 
626
  const verified = await new Promise<boolean>((resolve) => {
627
- const feedVad = (c: Buffer) => {
 
 
 
 
 
 
 
628
  if (this.vadInstance) {
629
  const float32 = new Float32Array(c.length / 2);
630
  for (let i = 0; i < float32.length; i++) {
@@ -637,15 +697,10 @@ class FriendService {
637
  const dataHandler = (chunk: Buffer) => {
638
  dataArrived = true;
639
  if (verifyTimer) { clearTimeout(verifyTimer); }
640
- // Forward first chunk to both STT and VAD
641
- onData(chunk);
642
- feedVad(chunk);
643
  // Swap to the permanent handler for subsequent chunks
644
  proc.stdout?.removeListener('data', dataHandler);
645
- proc.stdout?.on('data', (c: Buffer) => {
646
- onData(c);
647
- feedVad(c);
648
- });
649
  resolve(true);
650
  };
651
  proc.stdout?.on('data', dataHandler);
 
77
  private _flushing = false;
78
  /** Silero VAD instance (real ML-based voice activity detection) */
79
  private vadInstance: SileroVad | null = null;
80
+ /** When muted, audio from arecord is not forwarded to STT or VAD (prevents echo) */
81
+ private muted = false;
82
+ /** Timer to automatically unmute after estimated TTS playback */
83
+ private muteTimer: ReturnType<typeof setTimeout> | null = null;
84
 
85
  // ── React sync external store interface ──────────────────────────────
86
 
 
419
 
420
  this.capturing = false;
421
 
422
+ // Clear mute state
423
+ this.clearMute();
424
+
425
  // Reset VAD state silently (don't fire onSpeechEnd — we're finalizing below)
426
  if (this.vadInstance) {
427
  try {
 
503
  return transcript;
504
  }
505
 
506
+ /**
507
+ * Mute audio capture for an estimated duration to prevent TTS echo.
508
+ * Audio from arecord will not be forwarded to STT or VAD while muted.
509
+ * Automatically unmutes after the estimated TTS playback duration.
510
+ */
511
+ private muteForTts(text: string): void {
512
+ if (!this.capturing) return;
513
+
514
+ // Clear any existing mute timer
515
+ if (this.muteTimer) {
516
+ clearTimeout(this.muteTimer);
517
+ this.muteTimer = null;
518
+ }
519
+
520
+ this.muted = true;
521
+
522
+ // Pause VAD so it doesn't accumulate stale audio
523
+ this.vadInstance?.pause();
524
+
525
+ // Estimate TTS playback duration:
526
+ // Chinese ~8 chars/sec (125ms/char), English ~12 chars/sec (83ms/char)
527
+ // Avg ~100ms/char + 1s generation/network overhead + 0.5s margin
528
+ const estimatedMs = Math.max(3000, Math.round(text.length * 100) + 1500);
529
+
530
+ this.muteTimer = setTimeout(() => {
531
+ this.muted = false;
532
+ this.muteTimer = null;
533
+ // Resume VAD with fresh state (buffer cleared by pause())
534
+ this.vadInstance?.start();
535
+ }, estimatedMs);
536
+ }
537
+
538
+ /** Clear mute state immediately */
539
+ private clearMute(): void {
540
+ if (this.muteTimer) {
541
+ clearTimeout(this.muteTimer);
542
+ this.muteTimer = null;
543
+ }
544
+ this.muted = false;
545
+ }
546
+
547
  // ── Response broadcast (called by useFriendBridge) ───────────────────
548
 
549
  /**
 
566
  if (audioId) {
567
  const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
568
  broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
569
+
570
+ // Mute capture during TTS playback to prevent echo loop
571
+ if (this.capturing) {
572
+ this.muteForTts(text);
573
+ }
574
  }
575
  } catch (err) {
576
  console.warn('[FriendService] TTS generation failed:', err);
 
677
  let verifyTimer: ReturnType<typeof setTimeout> | null = null;
678
 
679
  const verified = await new Promise<boolean>((resolve) => {
680
+ const feedAudio = (c: Buffer) => {
681
+ // Skip when muted — prevents AI TTS echo from re-entering STT/VAD
682
+ if (this.muted) return;
683
+
684
+ // Forward to STT connection
685
+ onData(c);
686
+
687
+ // Forward to VAD for speech activity detection
688
  if (this.vadInstance) {
689
  const float32 = new Float32Array(c.length / 2);
690
  for (let i = 0; i < float32.length; i++) {
 
697
  const dataHandler = (chunk: Buffer) => {
698
  dataArrived = true;
699
  if (verifyTimer) { clearTimeout(verifyTimer); }
700
+ feedAudio(chunk);
 
 
701
  // Swap to the permanent handler for subsequent chunks
702
  proc.stdout?.removeListener('data', dataHandler);
703
+ proc.stdout?.on('data', feedAudio);
 
 
 
704
  resolve(true);
705
  };
706
  proc.stdout?.on('data', dataHandler);