fix: echo in VAD STT TTS
Browse files回声问题修复
新增静音机制 — 在 AI 通过 TTS 回答期间,暂时阻断麦克风音频进入 STT 和 VAD:
1. muted 标志 + muteTimer — 控制音频采集的静音状态
2. muteForTts(text) — 根据回复文本长度估算 TTS 播放时长,自动管理静音期:
- 估算公式:max(3000ms, text.length × 100ms + 1500ms)
- 中文字符约 125ms/字,英文约 83ms/字符,+1s 生成/网络开销 + 0.5s 余量
- 最短 3 秒
3. clearMute() — 用户按 F2 结束通话时立即清除静音状态
4. arecord 回调中的 feedAudio(c) — 静音时直接 return,音频不进入 STT 也不进入 VAD
完整流程:
用户说话 → 音频 → STT → 转录 → AI → broadcastResponse()
↓ TTS生成成功 + 采集进行中
muteForTts(text)
↓
静音期间所有音频丢弃
↓ (TTS播放完毕)
setTimeout → unmute
↓
恢复音频采集
- src/friend/FriendService.ts +63 -8
|
@@ -77,6 +77,10 @@ class FriendService {
|
|
| 77 |
private _flushing = false;
|
| 78 |
/** Silero VAD instance (real ML-based voice activity detection) */
|
| 79 |
private vadInstance: SileroVad | null = null;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
|
| 81 |
// ── React sync external store interface ──────────────────────────────
|
| 82 |
|
|
@@ -415,6 +419,9 @@ class FriendService {
|
|
| 415 |
|
| 416 |
this.capturing = false;
|
| 417 |
|
|
|
|
|
|
|
|
|
|
| 418 |
// Reset VAD state silently (don't fire onSpeechEnd — we're finalizing below)
|
| 419 |
if (this.vadInstance) {
|
| 420 |
try {
|
|
@@ -496,6 +503,47 @@ class FriendService {
|
|
| 496 |
return transcript;
|
| 497 |
}
|
| 498 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 499 |
// ── Response broadcast (called by useFriendBridge) ───────────────────
|
| 500 |
|
| 501 |
/**
|
|
@@ -518,6 +566,11 @@ class FriendService {
|
|
| 518 |
if (audioId) {
|
| 519 |
const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
|
| 520 |
broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 521 |
}
|
| 522 |
} catch (err) {
|
| 523 |
console.warn('[FriendService] TTS generation failed:', err);
|
|
@@ -624,7 +677,14 @@ class FriendService {
|
|
| 624 |
let verifyTimer: ReturnType<typeof setTimeout> | null = null;
|
| 625 |
|
| 626 |
const verified = await new Promise<boolean>((resolve) => {
|
| 627 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 628 |
if (this.vadInstance) {
|
| 629 |
const float32 = new Float32Array(c.length / 2);
|
| 630 |
for (let i = 0; i < float32.length; i++) {
|
|
@@ -637,15 +697,10 @@ class FriendService {
|
|
| 637 |
const dataHandler = (chunk: Buffer) => {
|
| 638 |
dataArrived = true;
|
| 639 |
if (verifyTimer) { clearTimeout(verifyTimer); }
|
| 640 |
-
|
| 641 |
-
onData(chunk);
|
| 642 |
-
feedVad(chunk);
|
| 643 |
// Swap to the permanent handler for subsequent chunks
|
| 644 |
proc.stdout?.removeListener('data', dataHandler);
|
| 645 |
-
proc.stdout?.on('data',
|
| 646 |
-
onData(c);
|
| 647 |
-
feedVad(c);
|
| 648 |
-
});
|
| 649 |
resolve(true);
|
| 650 |
};
|
| 651 |
proc.stdout?.on('data', dataHandler);
|
|
|
|
| 77 |
private _flushing = false;
|
| 78 |
/** Silero VAD instance (real ML-based voice activity detection) */
|
| 79 |
private vadInstance: SileroVad | null = null;
|
| 80 |
+
/** When muted, audio from arecord is not forwarded to STT or VAD (prevents echo) */
|
| 81 |
+
private muted = false;
|
| 82 |
+
/** Timer to automatically unmute after estimated TTS playback */
|
| 83 |
+
private muteTimer: ReturnType<typeof setTimeout> | null = null;
|
| 84 |
|
| 85 |
// ── React sync external store interface ──────────────────────────────
|
| 86 |
|
|
|
|
| 419 |
|
| 420 |
this.capturing = false;
|
| 421 |
|
| 422 |
+
// Clear mute state
|
| 423 |
+
this.clearMute();
|
| 424 |
+
|
| 425 |
// Reset VAD state silently (don't fire onSpeechEnd — we're finalizing below)
|
| 426 |
if (this.vadInstance) {
|
| 427 |
try {
|
|
|
|
| 503 |
return transcript;
|
| 504 |
}
|
| 505 |
|
| 506 |
+
/**
|
| 507 |
+
* Mute audio capture for an estimated duration to prevent TTS echo.
|
| 508 |
+
* Audio from arecord will not be forwarded to STT or VAD while muted.
|
| 509 |
+
* Automatically unmutes after the estimated TTS playback duration.
|
| 510 |
+
*/
|
| 511 |
+
private muteForTts(text: string): void {
|
| 512 |
+
if (!this.capturing) return;
|
| 513 |
+
|
| 514 |
+
// Clear any existing mute timer
|
| 515 |
+
if (this.muteTimer) {
|
| 516 |
+
clearTimeout(this.muteTimer);
|
| 517 |
+
this.muteTimer = null;
|
| 518 |
+
}
|
| 519 |
+
|
| 520 |
+
this.muted = true;
|
| 521 |
+
|
| 522 |
+
// Pause VAD so it doesn't accumulate stale audio
|
| 523 |
+
this.vadInstance?.pause();
|
| 524 |
+
|
| 525 |
+
// Estimate TTS playback duration:
|
| 526 |
+
// Chinese ~8 chars/sec (125ms/char), English ~12 chars/sec (83ms/char)
|
| 527 |
+
// Avg ~100ms/char + 1s generation/network overhead + 0.5s margin
|
| 528 |
+
const estimatedMs = Math.max(3000, Math.round(text.length * 100) + 1500);
|
| 529 |
+
|
| 530 |
+
this.muteTimer = setTimeout(() => {
|
| 531 |
+
this.muted = false;
|
| 532 |
+
this.muteTimer = null;
|
| 533 |
+
// Resume VAD with fresh state (buffer cleared by pause())
|
| 534 |
+
this.vadInstance?.start();
|
| 535 |
+
}, estimatedMs);
|
| 536 |
+
}
|
| 537 |
+
|
| 538 |
+
/** Clear mute state immediately */
|
| 539 |
+
private clearMute(): void {
|
| 540 |
+
if (this.muteTimer) {
|
| 541 |
+
clearTimeout(this.muteTimer);
|
| 542 |
+
this.muteTimer = null;
|
| 543 |
+
}
|
| 544 |
+
this.muted = false;
|
| 545 |
+
}
|
| 546 |
+
|
| 547 |
// ── Response broadcast (called by useFriendBridge) ───────────────────
|
| 548 |
|
| 549 |
/**
|
|
|
|
| 566 |
if (audioId) {
|
| 567 |
const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
|
| 568 |
broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
|
| 569 |
+
|
| 570 |
+
// Mute capture during TTS playback to prevent echo loop
|
| 571 |
+
if (this.capturing) {
|
| 572 |
+
this.muteForTts(text);
|
| 573 |
+
}
|
| 574 |
}
|
| 575 |
} catch (err) {
|
| 576 |
console.warn('[FriendService] TTS generation failed:', err);
|
|
|
|
| 677 |
let verifyTimer: ReturnType<typeof setTimeout> | null = null;
|
| 678 |
|
| 679 |
const verified = await new Promise<boolean>((resolve) => {
|
| 680 |
+
const feedAudio = (c: Buffer) => {
|
| 681 |
+
// Skip when muted — prevents AI TTS echo from re-entering STT/VAD
|
| 682 |
+
if (this.muted) return;
|
| 683 |
+
|
| 684 |
+
// Forward to STT connection
|
| 685 |
+
onData(c);
|
| 686 |
+
|
| 687 |
+
// Forward to VAD for speech activity detection
|
| 688 |
if (this.vadInstance) {
|
| 689 |
const float32 = new Float32Array(c.length / 2);
|
| 690 |
for (let i = 0; i < float32.length; i++) {
|
|
|
|
| 697 |
const dataHandler = (chunk: Buffer) => {
|
| 698 |
dataArrived = true;
|
| 699 |
if (verifyTimer) { clearTimeout(verifyTimer); }
|
| 700 |
+
feedAudio(chunk);
|
|
|
|
|
|
|
| 701 |
// Swap to the permanent handler for subsequent chunks
|
| 702 |
proc.stdout?.removeListener('data', dataHandler);
|
| 703 |
+
proc.stdout?.on('data', feedAudio);
|
|
|
|
|
|
|
|
|
|
| 704 |
resolve(true);
|
| 705 |
};
|
| 706 |
proc.stdout?.on('data', dataHandler);
|