chenbhao commited on
Commit
2578c9e
·
1 Parent(s): 54d2210

fix: tools call or think voice play will stop receive audio

Browse files

用户说话 → VAD检测说话结束 → _flushVadSegment()
→ sendText(transcript) → 进入CLI对话
→ startAiTurnMute()
├─ muted = true
├─ VAD pause()
└─ 30秒超时(安全网,覆盖AI处理、工具调用、思考等)

AI处理中(WebSearch / 深度思考 / 等) → 音频静音,不收不送

AI回复就绪 → broadcastResponse()
→ TTS生成成功
→ extendMuteForTts(audioId)
├─ 清除30秒安全网
├─ 解析MP3算出精确音频时长
└─ 新定时器 = 精确时长

TTS播放中 → 音频静音

TTS播放完毕 → unmute()
├─ muted = false
├─ VAD start() → 恢复收音
└─ 下一轮对话开始

30 秒兜底防止 AI 不回复(错误等)导致永久静音,extendMuteForTts 在 TTS 生成后会精确定时到音频实际长度

Files changed (1) hide show
  1. src/friend/FriendService.ts +45 -26
src/friend/FriendService.ts CHANGED
@@ -289,6 +289,12 @@ class FriendService {
289
  if (transcript) {
290
  this.captureTranscripts = [];
291
  this.sendText(transcript);
 
 
 
 
 
 
292
  }
293
  } catch (err) {
294
  console.error('[FriendService] _flushVadSegment error:', err);
@@ -505,37 +511,52 @@ class FriendService {
505
  }
506
 
507
  /**
508
- * Mute audio capture for the exact duration of the TTS audio file.
509
- * Audio from arecord will not be forwarded to STT or VAD while muted.
510
- * Automatically unmutes when the TTS audio would have finished playing.
 
 
 
 
511
  */
512
- private muteForTts(audioId: string, text: string): void {
513
  if (!this.capturing) return;
514
 
515
- // Clear any existing mute timer
516
- if (this.muteTimer) {
517
- clearTimeout(this.muteTimer);
518
- this.muteTimer = null;
519
- }
520
 
521
  this.muted = true;
522
-
523
- // Pause VAD so it doesn't accumulate stale audio
524
  this.vadInstance?.pause();
525
 
526
- // Parse the actual MP3 duration for precise mute timing
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
527
  let muteMs = this.getMp3DurationMs(audioId);
528
- if (muteMs <= 0) {
529
- // Fallback estimate if parsing fails
530
- muteMs = Math.max(3000, Math.round(text.length * 100) + 1500);
531
- }
532
 
533
- this.muteTimer = setTimeout(() => {
534
- this.muted = false;
535
- this.muteTimer = null;
536
- // Resume VAD with fresh state (buffer cleared by pause())
537
- this.vadInstance?.start();
538
- }, muteMs);
 
 
539
  }
540
 
541
  /** Parse MP3 file to get exact audio duration in milliseconds.
@@ -647,10 +668,8 @@ class FriendService {
647
  const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
648
  broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
649
 
650
- // Mute capture for the exact TTS audio duration
651
- if (this.capturing) {
652
- this.muteForTts(audioId, text);
653
- }
654
  }
655
  } catch (err) {
656
  console.warn('[FriendService] TTS generation failed:', err);
 
289
  if (transcript) {
290
  this.captureTranscripts = [];
291
  this.sendText(transcript);
292
+
293
+ // Mute the entire AI turn: from transcript submission → AI processing
294
+ // (tools, deep thinking) → response → TTS playback.
295
+ // This prevents both TTS echo AND capturing accidental speech during
296
+ // AI processing (e.g. "hmm", "ok").
297
+ this.startAiTurnMute();
298
  }
299
  } catch (err) {
300
  console.error('[FriendService] _flushVadSegment error:', err);
 
511
  }
512
 
513
  /**
514
+ * Start mute at the beginning of an AI turn (when user speech is submitted).
515
+ *
516
+ * Audio from arecord is blocked from STT/VAD during:
517
+ * AI processing (tools, deep thinking) → response generation → TTS playback
518
+ *
519
+ * A long timeout (30s) covers the AI processing window without estimation.
520
+ * The timer is later refined by extendMuteForTts() when TTS audio is ready.
521
  */
522
+ private startAiTurnMute(): void {
523
  if (!this.capturing) return;
524
 
525
+ // Clear any existing timer (from a previous turn)
526
+ if (this.muteTimer) clearTimeout(this.muteTimer);
 
 
 
527
 
528
  this.muted = true;
 
 
529
  this.vadInstance?.pause();
530
 
531
+ // 30s covers almost all AI response cycles (tools, deep thinking, etc.).
532
+ // The timer is reset in extendMuteForTts() when TTS duration is known.
533
+ this.muteTimer = setTimeout(() => this.unmute(), 30_000);
534
+ }
535
+
536
+ /**
537
+ * Refine the mute timer to match actual TTS audio duration once it's ready.
538
+ * Called from broadcastResponse() after TTS generation succeeds.
539
+ * Resets the timer to exactly the audio playback length.
540
+ */
541
+ private extendMuteForTts(audioId: string): void {
542
+ if (!this.capturing) return;
543
+ if (!this.muted) return; // turn already ended, don't re-mute
544
+
545
+ // Clear the generous timer from startAiTurnMute
546
+ if (this.muteTimer) clearTimeout(this.muteTimer);
547
+
548
+ // Parse exact audio duration from the MP3 file
549
  let muteMs = this.getMp3DurationMs(audioId);
550
+ if (muteMs <= 0) muteMs = 3000; // safety fallback
 
 
 
551
 
552
+ this.muteTimer = setTimeout(() => this.unmute(), muteMs);
553
+ }
554
+
555
+ /** Unmute and resume VAD */
556
+ private unmute(): void {
557
+ this.muted = false;
558
+ this.muteTimer = null;
559
+ this.vadInstance?.start();
560
  }
561
 
562
  /** Parse MP3 file to get exact audio duration in milliseconds.
 
668
  const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
669
  broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
670
 
671
+ // Refine mute timer to exact TTS audio duration
672
+ this.extendMuteForTts(audioId);
 
 
673
  }
674
  } catch (err) {
675
  console.warn('[FriendService] TTS generation failed:', err);