fix: tools call or think voice play will stop receive audio
Browse files用户说话 → VAD检测说话结束 → _flushVadSegment()
→ sendText(transcript) → 进入CLI对话
→ startAiTurnMute()
├─ muted = true
├─ VAD pause()
└─ 30秒超时(安全网,覆盖AI处理、工具调用、思考等)
AI处理中(WebSearch / 深度思考 / 等) → 音频静音,不收不送
AI回复就绪 → broadcastResponse()
→ TTS生成成功
→ extendMuteForTts(audioId)
├─ 清除30秒安全网
├─ 解析MP3算出精确音频时长
└─ 新定时器 = 精确时长
TTS播放中 → 音频静音
TTS播放完毕 → unmute()
├─ muted = false
├─ VAD start() → 恢复收音
└─ 下一轮对话开始
30 秒兜底防止 AI 不回复(错误等)导致永久静音,extendMuteForTts 在 TTS 生成后会精确定时到音频实际长度
- src/friend/FriendService.ts +45 -26
src/friend/FriendService.ts
CHANGED
|
@@ -289,6 +289,12 @@ class FriendService {
|
|
| 289 |
if (transcript) {
|
| 290 |
this.captureTranscripts = [];
|
| 291 |
this.sendText(transcript);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
}
|
| 293 |
} catch (err) {
|
| 294 |
console.error('[FriendService] _flushVadSegment error:', err);
|
|
@@ -505,37 +511,52 @@ class FriendService {
|
|
| 505 |
}
|
| 506 |
|
| 507 |
/**
|
| 508 |
-
*
|
| 509 |
-
*
|
| 510 |
-
*
|
|
|
|
|
|
|
|
|
|
|
|
|
| 511 |
*/
|
| 512 |
-
private
|
| 513 |
if (!this.capturing) return;
|
| 514 |
|
| 515 |
-
// Clear any existing
|
| 516 |
-
if (this.muteTimer)
|
| 517 |
-
clearTimeout(this.muteTimer);
|
| 518 |
-
this.muteTimer = null;
|
| 519 |
-
}
|
| 520 |
|
| 521 |
this.muted = true;
|
| 522 |
-
|
| 523 |
-
// Pause VAD so it doesn't accumulate stale audio
|
| 524 |
this.vadInstance?.pause();
|
| 525 |
|
| 526 |
-
//
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 527 |
let muteMs = this.getMp3DurationMs(audioId);
|
| 528 |
-
if (muteMs <= 0)
|
| 529 |
-
// Fallback estimate if parsing fails
|
| 530 |
-
muteMs = Math.max(3000, Math.round(text.length * 100) + 1500);
|
| 531 |
-
}
|
| 532 |
|
| 533 |
-
this.muteTimer = setTimeout(() =>
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
|
|
|
|
|
|
| 539 |
}
|
| 540 |
|
| 541 |
/** Parse MP3 file to get exact audio duration in milliseconds.
|
|
@@ -647,10 +668,8 @@ class FriendService {
|
|
| 647 |
const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
|
| 648 |
broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
|
| 649 |
|
| 650 |
-
//
|
| 651 |
-
|
| 652 |
-
this.muteForTts(audioId, text);
|
| 653 |
-
}
|
| 654 |
}
|
| 655 |
} catch (err) {
|
| 656 |
console.warn('[FriendService] TTS generation failed:', err);
|
|
|
|
| 289 |
if (transcript) {
|
| 290 |
this.captureTranscripts = [];
|
| 291 |
this.sendText(transcript);
|
| 292 |
+
|
| 293 |
+
// Mute the entire AI turn: from transcript submission → AI processing
|
| 294 |
+
// (tools, deep thinking) → response → TTS playback.
|
| 295 |
+
// This prevents both TTS echo AND capturing accidental speech during
|
| 296 |
+
// AI processing (e.g. "hmm", "ok").
|
| 297 |
+
this.startAiTurnMute();
|
| 298 |
}
|
| 299 |
} catch (err) {
|
| 300 |
console.error('[FriendService] _flushVadSegment error:', err);
|
|
|
|
| 511 |
}
|
| 512 |
|
| 513 |
/**
|
| 514 |
+
* Start mute at the beginning of an AI turn (when user speech is submitted).
|
| 515 |
+
*
|
| 516 |
+
* Audio from arecord is blocked from STT/VAD during:
|
| 517 |
+
* AI processing (tools, deep thinking) → response generation → TTS playback
|
| 518 |
+
*
|
| 519 |
+
* A long timeout (30s) covers the AI processing window without estimation.
|
| 520 |
+
* The timer is later refined by extendMuteForTts() when TTS audio is ready.
|
| 521 |
*/
|
| 522 |
+
private startAiTurnMute(): void {
|
| 523 |
if (!this.capturing) return;
|
| 524 |
|
| 525 |
+
// Clear any existing timer (from a previous turn)
|
| 526 |
+
if (this.muteTimer) clearTimeout(this.muteTimer);
|
|
|
|
|
|
|
|
|
|
| 527 |
|
| 528 |
this.muted = true;
|
|
|
|
|
|
|
| 529 |
this.vadInstance?.pause();
|
| 530 |
|
| 531 |
+
// 30s covers almost all AI response cycles (tools, deep thinking, etc.).
|
| 532 |
+
// The timer is reset in extendMuteForTts() when TTS duration is known.
|
| 533 |
+
this.muteTimer = setTimeout(() => this.unmute(), 30_000);
|
| 534 |
+
}
|
| 535 |
+
|
| 536 |
+
/**
|
| 537 |
+
* Refine the mute timer to match actual TTS audio duration once it's ready.
|
| 538 |
+
* Called from broadcastResponse() after TTS generation succeeds.
|
| 539 |
+
* Resets the timer to exactly the audio playback length.
|
| 540 |
+
*/
|
| 541 |
+
private extendMuteForTts(audioId: string): void {
|
| 542 |
+
if (!this.capturing) return;
|
| 543 |
+
if (!this.muted) return; // turn already ended, don't re-mute
|
| 544 |
+
|
| 545 |
+
// Clear the generous timer from startAiTurnMute
|
| 546 |
+
if (this.muteTimer) clearTimeout(this.muteTimer);
|
| 547 |
+
|
| 548 |
+
// Parse exact audio duration from the MP3 file
|
| 549 |
let muteMs = this.getMp3DurationMs(audioId);
|
| 550 |
+
if (muteMs <= 0) muteMs = 3000; // safety fallback
|
|
|
|
|
|
|
|
|
|
| 551 |
|
| 552 |
+
this.muteTimer = setTimeout(() => this.unmute(), muteMs);
|
| 553 |
+
}
|
| 554 |
+
|
| 555 |
+
/** Unmute and resume VAD */
|
| 556 |
+
private unmute(): void {
|
| 557 |
+
this.muted = false;
|
| 558 |
+
this.muteTimer = null;
|
| 559 |
+
this.vadInstance?.start();
|
| 560 |
}
|
| 561 |
|
| 562 |
/** Parse MP3 file to get exact audio duration in milliseconds.
|
|
|
|
| 668 |
const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
|
| 669 |
broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
|
| 670 |
|
| 671 |
+
// Refine mute timer to exact TTS audio duration
|
| 672 |
+
this.extendMuteForTts(audioId);
|
|
|
|
|
|
|
| 673 |
}
|
| 674 |
} catch (err) {
|
| 675 |
console.warn('[FriendService] TTS generation failed:', err);
|