feat: VAD but just case choose

bun.lock

@@ -102,6 +102,7 @@
         "vscode-jsonrpc": "^8.2.1",
         "vscode-languageserver-protocol": "^3.17.5",
         "vscode-languageserver-types": "^3.17.5",
+        "webrtcvad": "^1.0.1",
         "wrap-ansi": "^10.0.0",
         "ws": "^8.20.0",
         "xss": "^1.0.15",
@@ -529,6 +530,8 @@
 
     "bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="],
 
+    "bindings": ["bindings@1.5.0", "", { "dependencies": { "file-uri-to-path": "1.0.0" } }, "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ=="],
+
     "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
 
     "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
@@ -681,6 +684,8 @@
 
     "figures": ["figures@6.1.0", "", { "dependencies": { "is-unicode-supported": "^2.0.0" } }, "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg=="],
 
+    "file-uri-to-path": ["file-uri-to-path@1.0.0", "", {}, "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="],
+
     "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="],
 
     "find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="],
@@ -877,6 +882,8 @@
 
     "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
 
+    "node-addon-api": ["node-addon-api@1.7.2", "", {}, "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg=="],
+
     "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
 
     "node-edge-tts": ["node-edge-tts@1.2.10", "", { "dependencies": { "https-proxy-agent": "^7.0.1", "ws": "^8.13.0", "yargs": "^17.7.2" }, "bin": { "node-edge-tts": "bin.js" } }, "sha512-bV2i4XU54D45+US0Zm1HcJRkifuB3W438dWyuJEHLQdKxnuqlI1kim2MOvR6Q3XUQZvfF9PoDyR1Rt7aeXhPdQ=="],
@@ -1093,6 +1100,8 @@
 
     "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
 
+    "webrtcvad": ["webrtcvad@1.0.1", "", { "dependencies": { "bindings": "^1.3.0", "node-addon-api": "^1.7.1" } }, "sha512-oLfReCmGMpRducFWKP+o0GpKZLPj0u6qkln3P7wGaEzyjxtBiFuzvK28pmKF5SSihEkC1RGKO7Pi9C4VfY1q4Q=="],
+
     "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
 
     "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],

package.json

@@ -120,6 +120,7 @@
     "vscode-jsonrpc": "^8.2.1",
     "vscode-languageserver-protocol": "^3.17.5",
     "vscode-languageserver-types": "^3.17.5",
+    "webrtcvad": "^1.0.1",
     "wrap-ansi": "^10.0.0",
     "ws": "^8.20.0",
     "xss": "^1.0.15",

src/components/friend/frontend/components/ChatInput.tsx

@@ -15,8 +15,6 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
   const [menuOpen, setMenuOpen] = useState(false)
   const [voiceCallActive, setVoiceCallActive] = useState(false)
   const inputRef = useRef<HTMLInputElement>(null)
-  const silenceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
-  const lastSentIndexRef = useRef(0)
   const voiceCallActiveRef = useRef(false)
   const endVoiceCallRef = useRef<() => void>(() => {})
   const serverStt = useServerStt()
@@ -79,13 +77,6 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
     if (recording) stopRecording()
   }, [recording, stopRecording])
 
-  // --- Voice Call: constants ---
-  const HARD_PUNCT = /[。！？\.\!\?]$/
-  const SOFT_PUNCT = /[，、；,;：:]$/
-  const SOFT_PUNCT_MIN_LEN = 10
-  const MAX_UNSENT_LEN = 30
-  const SILENCE_SEND_MS = 1200
-
   // --- Voice Call: delayed TTS interrupt ---
   const interruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
   const scheduleInterrupt = useCallback(() => {
@@ -99,90 +90,38 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
     if (interruptTimerRef.current) { clearTimeout(interruptTimerRef.current); interruptTimerRef.current = null }
   }, [])
 
-  // --- Voice Call: send + mute capture for 3s after send ---
-  const VOICE_MUTE_AFTER_SEND_MS = 2000
-  const mutedUntilRef = useRef(0)
-
-  const voiceCallSend = useCallback(async (msg: string) => {
-    const trimmed = msg.trim()
-    if (!trimmed) return
-    mutedUntilRef.current = Date.now() + VOICE_MUTE_AFTER_SEND_MS
-    cancelInterrupt()
-    ;(window as any).__clawInterruptAudio?.()
-    try {
-      await fetch(`${FRIEND_API}/chat`, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ message: trimmed }),
-      })
-    } catch (err) {
-      console.error('Voice call send failed:', err)
-    }
-  }, [cancelInterrupt])
-
-  // --- Voice Call: smart sentence segmentation ---
-  const handleVoiceCallResult = useCallback((fullTranscript: string, isFinal: boolean) => {
-    if (Date.now() < mutedUntilRef.current) {
-      lastSentIndexRef.current = fullTranscript.length
-      return
-    }
-
-    const unsent = fullTranscript.slice(lastSentIndexRef.current)
-    setText(unsent)
-    onActiveChange?.(unsent.length > 0)
-
-    if (silenceTimerRef.current) { clearTimeout(silenceTimerRef.current); silenceTimerRef.current = null }
-
-    if (isFinal) {
-      const shouldSend =
-        HARD_PUNCT.test(unsent) ||
-        (SOFT_PUNCT.test(unsent) && unsent.length >= SOFT_PUNCT_MIN_LEN) ||
-        unsent.length >= MAX_UNSENT_LEN
-
-      if (shouldSend) {
-        voiceCallSend(unsent)
-        lastSentIndexRef.current = fullTranscript.length
-        setText('')
-        onActiveChange?.(false)
-        return
-      }
-    }
-
-    if (unsent.trim()) {
-      silenceTimerRef.current = setTimeout(() => {
-        if (!voiceCallActiveRef.current) return
-        const chunk = unsent.trim()
-        if (chunk) {
-          voiceCallSend(chunk)
-          lastSentIndexRef.current = fullTranscript.length
-          setText('')
-          onActiveChange?.(false)
-        }
-      }, SILENCE_SEND_MS)
-    }
-  }, [voiceCallSend, onActiveChange])
+  // Voice call display timer — clear text after 3s
+  const displayTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
 
-  // --- Voice Call: start (always server-side, no getUserMedia) ---
+  // --- Voice Call: start (browser VAD with @ricky0123/vad-web) ---
   const startVoiceCall = useCallback(async () => {
     setVoiceCallActive(true)
     voiceCallActiveRef.current = true
-    lastSentIndexRef.current = 0
     setText('')
     setOpen(true)
 
-    // Always use server-side STT via HTTP polling.
-    // Audio capture is done server-side via cpal (in-process native addon)
-    // so no getUserMedia call is needed on the frontend — this avoids
-    // WebKitGTK permission issues on Linux.
     serverStt.startStreaming(
       sttProvider,
-      (text, isFinal) => {
-        handleVoiceCallResult(text, isFinal)
+      (text, _isFinal) => {
+        // Server-side transcribeAudioSegment() already calls sendText()
+        // to enqueue the message into the CLI conversation.
+        // We only need to display the transcript in the input bar briefly.
+        if (text.trim()) {
+          setText(text)
+          onActiveChange?.(true)
+          // Auto-clear after 3s
+          if (displayTimerRef.current) clearTimeout(displayTimerRef.current)
+          displayTimerRef.current = setTimeout(() => {
+            if (voiceCallActiveRef.current) {
+              setText('')
+              onActiveChange?.(false)
+            }
+          }, 3000)
+        }
       },
       (err) => {
-        console.error('Server STT error:', err)
+        console.error('Browser VAD error:', err)
         if (voiceCallActiveRef.current) {
-          // Show error text in the input bar, then end call after 3s
           setText(`语音启动失败: ${err}`)
           setTimeout(() => {
             setText('')
@@ -193,7 +132,7 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
       language === 'en' ? 'en' : 'zh',
     )
     setRecording(true)
-  }, [handleVoiceCallResult, sttProvider, serverStt, language])
+  }, [sttProvider, serverStt, language, onActiveChange])
 
   // --- Voice Call: end ---
   const endVoiceCall = useCallback(async () => {
@@ -202,22 +141,21 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
     setVoiceCallActive(false)
     setRecording(false)
 
-    // Stop capture and get the final transcript
-    const text = await serverStt.stopStreaming()
+    // Stop VAD and clean up
+    const finalText = await serverStt.stopStreaming()
 
-    // Send any remaining transcript as a message (no-op if empty)
-    if (text.trim()) {
-      voiceCallSend(text)
+    // Send any remaining transcript (no-op if empty — segments were already
+    // sent to CLI by transcribeAudioSegment() during the call).
+    if (finalText.trim()) {
+      send(finalText)
     }
 
-    if (silenceTimerRef.current) { clearTimeout(silenceTimerRef.current); silenceTimerRef.current = null }
     cancelInterrupt()
-    mutedUntilRef.current = 0
 
     setText('')
     setOpen(false)
     onActiveChange?.(false)
-  }, [onActiveChange, cancelInterrupt, serverStt, voiceCallSend])
+  }, [onActiveChange, cancelInterrupt, serverStt, send])
 
   endVoiceCallRef.current = endVoiceCall
 

src/components/friend/frontend/hooks/useServerStt.ts

@@ -1,17 +1,14 @@
 /**
- * Hook for using server-side STT (Groq Whisper, Anthropic Voice Stream, Local Whisper, Doubao)
+ * Hook for using browser-based energy VAD + server-side STT
  * in the Friend Tauri app.
  *
- * Audio capture is done server-side via cpal (in-process native addon)
- * so no getUserMedia call is needed on the frontend — this avoids
- * WebKitGTK permission issues on Linux.
+ * Audio capture and VAD run in the browser via Web Audio API.
+ * Uses simple energy (RMS) detection for voice activity —
+ * no WASM or external dependencies needed.
  *
  * Two modes:
  *   - push-to-talk: POST /voice/start → capture → POST /voice/stop → get text
- *   - streaming:    POST /voice/start → poll /voice/status + auto-cycle every 3s
- *
- * The auto-cycle in streaming mode ensures batch STT providers (Groq, local Whisper)
- * produce regular final transcripts even though they don't support interim results.
+ *   - voice-call:   Browser energy VAD detects speech segments → sends to server for STT
  */
 import { useRef, useCallback, useState } from 'react'
 
@@ -19,11 +16,153 @@ const FRIEND_API_BASE = 'http://127.0.0.1:3456/plugins/friend'
 
 export type SttProvider = 'browser' | 'groq' | 'anthropic' | 'local' | 'doubao'
 
+/**
+ * Convert Float32Array audio (between -1 and 1, 16kHz) to a WAV Blob.
+ */
+function float32ToWav(samples: Float32Array, sampleRate = 16000): Blob {
+  const numChannels = 1
+  const bitsPerSample = 16
+  const byteRate = sampleRate * numChannels * (bitsPerSample / 8)
+  const blockAlign = numChannels * (bitsPerSample / 8)
+  const dataSize = samples.length * (bitsPerSample / 8)
+  const headerSize = 44
+  const totalSize = headerSize + dataSize
+
+  const buffer = new ArrayBuffer(totalSize)
+  const view = new DataView(buffer)
+
+  // RIFF header
+  writeString(view, 0, 'RIFF')
+  view.setUint32(4, totalSize - 8, true)
+  writeString(view, 8, 'WAVE')
+
+  // fmt chunk
+  writeString(view, 12, 'fmt ')
+  view.setUint32(16, 16, true)
+  view.setUint16(20, 1, true) // PCM
+  view.setUint16(22, numChannels, true)
+  view.setUint32(24, sampleRate, true)
+  view.setUint32(28, byteRate, true)
+  view.setUint16(32, blockAlign, true)
+  view.setUint16(34, bitsPerSample, true)
+
+  // data chunk
+  writeString(view, 36, 'data')
+  view.setUint32(40, dataSize, true)
+
+  // Write PCM samples
+  let offset = 44
+  for (let i = 0; i < samples.length; i++) {
+    const s = Math.max(-1, Math.min(1, samples[i]))
+    view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true)
+    offset += 2
+  }
+
+  return new Blob([buffer], { type: 'audio/wav' })
+}
+
+function writeString(view: DataView, offset: number, str: string): void {
+  for (let i = 0; i < str.length; i++) {
+    view.setUint8(offset + i, str.charCodeAt(i))
+  }
+}
+
+/** Compute RMS (root mean square) energy of a float32 audio frame */
+function computeRms(samples: Float32Array): number {
+  let sum = 0
+  for (let i = 0; i < samples.length; i++) {
+    sum += samples[i] * samples[i]
+  }
+  return Math.sqrt(sum / samples.length)
+}
+
+/**
+ * Simple energy-based Voice Activity Detector.
+ * Uses RMS threshold with configurable hangover time.
+ */
+class EnergyVad {
+  private threshold: number
+  private hangoverFrames: number
+  private speaking = false
+  private silentFrames = 0
+  private speechFrames: Float32Array[] = []
+  private speechStartRms = 0
+
+  constructor(threshold = 0.008, hangoverMs = 800, frameMs = 30) {
+    this.threshold = threshold
+    this.hangoverFrames = Math.ceil(hangoverMs / frameMs)
+  }
+
+  /** Process a single audio frame. Returns true if speech just ended (with accumulated audio). */
+  process(frame: Float32Array, rms?: number): { speechEnded: boolean; audio: Float32Array | null } {
+    const energy = rms ?? computeRms(frame)
+
+    if (energy > this.threshold && !this.speaking) {
+      // Speech just started
+      this.speaking = true
+      this.silentFrames = 0
+      this.speechFrames = [frame]
+      this.speechStartRms = energy
+      return { speechEnded: false, audio: null }
+    }
+
+    if (this.speaking) {
+      this.speechFrames.push(frame)
+
+      if (energy < this.threshold) {
+        this.silentFrames++
+        if (this.silentFrames >= this.hangoverFrames) {
+          // Speech ended after hangover
+          const audio = this.flush()
+          return { speechEnded: true, audio }
+        }
+      } else {
+        this.silentFrames = 0
+      }
+    }
+
+    return { speechEnded: false, audio: null }
+  }
+
+  /** Flush accumulated audio and reset state */
+  private flush(): Float32Array | null {
+    if (this.speechFrames.length < 3) {
+      this.speechFrames = []
+      this.speaking = false
+      return null
+    }
+    const totalLen = this.speechFrames.reduce((s, f) => s + f.length, 0)
+    const merged = new Float32Array(totalLen)
+    let offset = 0
+    for (const f of this.speechFrames) {
+      merged.set(f, offset)
+      offset += f.length
+    }
+    this.speechFrames = []
+    this.speaking = false
+    return merged
+  }
+
+  reset(): void {
+    this.speaking = false
+    this.silentFrames = 0
+    this.speechFrames = []
+  }
+
+  isSpeaking(): boolean {
+    return this.speaking
+  }
+}
+
 export function useServerStt() {
   const [connected, setConnected] = useState(false)
-  const cycleTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
-  const cycleAccumRef = useRef('')
-  const streamingGenRef = useRef(0) // generation counter to prevent stale callbacks
+  const onTranscriptRef = useRef<((text: string, isFinal: boolean) => void) | null>(null)
+  const onErrorRef = useRef<((err: string) => void) | null>(null)
+  const vadRef = useRef<EnergyVad | null>(null)
+  const audioContextRef = useRef<AudioContext | null>(null)
+  const streamRef = useRef<MediaStream | null>(null)
+  const sourceNodeRef = useRef<MediaStreamAudioSourceNode | null>(null)
+  const scriptNodeRef = useRef<ScriptProcessorNode | null>(null)
 
   // ── Push-to-talk ──────────────────────────────────────────────────────
 
@@ -53,48 +192,16 @@ export function useServerStt() {
     return data.text || ''
   }, [])
 
-  // ── Streaming (voice call) with auto-cycle ────────────────────────────
-
-  const doCycle = useCallback(
-    (onTranscript: (text: string, isFinal: boolean) => void, gen: number): void => {
-      const isStale = () => streamingGenRef.current !== gen
-      fetch(`${FRIEND_API_BASE}/voice/stop`, { method: 'POST' })
-        .then(async (res) => {
-          if (isStale()) return
-          if (!res.ok) return
-          const data = await res.json()
-          if (data.text?.trim()) {
-            cycleAccumRef.current += data.text
-            onTranscript(cycleAccumRef.current, true)
-          }
-        })
-        .then(() => {
-          if (isStale()) return
-          // Restart capture for next cycle
-          return fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
-        })
-        .then(async (res) => {
-          if (isStale()) return
-          if (!res?.ok) {
-            setConnected(false)
-            return
-          }
-          setConnected(true)
-          // Schedule next cycle in 3s
-          cycleTimerRef.current = setTimeout(
-            () => doCycle(onTranscript, gen),
-            3000,
-          )
-        })
-        .catch(() => {})
-    },
-    [],
-  )
+  // ── Voice call with browser energy VAD ────────────────────────────────
 
   /**
-   * Start streaming STT for voice call mode.
-   * Polls interim results every 500ms AND auto-cycles every 3s to get
-   * final transcripts from batch STT providers (Groq, local Whisper).
+   * Start voice call mode with browser-based energy VAD.
+   *
+   * Uses Web Audio API (ScriptProcessorNode) to detect speech segments
+   * via RMS threshold. No WASM or external dependencies.
+   *
+   * When a speech segment ends, the audio is sent to the server for STT
+   * transcription (same /voice/stt-segment endpoint).
    */
   const startStreaming = useCallback(
     (
@@ -103,80 +210,115 @@ export function useServerStt() {
       onError: (err: string) => void,
       _language = 'zh',
     ) => {
-      // Clean up any existing state and bump generation
-      cycleAccumRef.current = ''
-      const myGen = ++streamingGenRef.current
-      const existingPollId = (window as any).__friendSttPollId
-      if (existingPollId) { clearInterval(existingPollId); delete (window as any).__friendSttPollId }
-      if (cycleTimerRef.current) { clearTimeout(cycleTimerRef.current); cycleTimerRef.current = null }
-
-      fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
-        .then(async (res) => {
-          if (streamingGenRef.current !== myGen) return
-          if (!res.ok) {
-            const body = await res.json().catch(() => ({}))
-            throw new Error(body.error || 'STT start failed')
-          }
-          setConnected(true)
+      // Clean up any existing session
+      stopCapture()
+
+      onTranscriptRef.current = onTranscript
+      onErrorRef.current = onError
+
+      const startCapture = async () => {
+        try {
+          const stream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+              channelCount: 1,
+              echoCancellation: true,
+              autoGainControl: true,
+              noiseSuppression: true,
+              sampleRate: 16000,
+            },
+          })
+          streamRef.current = stream
+
+          const audioCtx = new AudioContext({ sampleRate: 16000 })
+          audioContextRef.current = audioCtx
+
+          const source = audioCtx.createMediaStreamSource(stream)
+          sourceNodeRef.current = source
+
+          const scriptNode = audioCtx.createScriptProcessor(1024, 1, 1)
+          scriptNodeRef.current = scriptNode
+
+          const vad = new EnergyVad(0.006, 800, 64)
+          vadRef.current = vad
+
+          scriptNode.onaudioprocess = (e) => {
+            if (!vadRef.current) return
+
+            const input = e.inputBuffer.getChannelData(0)
+            const rms = computeRms(input)
+            const result = vadRef.current.process(input, rms)
+
+            if (result.speechEnded && result.audio && result.audio.length > 0) {
+              // Send to server for STT
+              const wavBlob = float32ToWav(result.audio)
 
-          // Poll for interim results every 500ms (works with streaming STT providers)
-          const pollId = setInterval(async () => {
-            try {
-              if (streamingGenRef.current !== myGen) { clearInterval(pollId); return }
-              const statusRes = await fetch(`${FRIEND_API_BASE}/voice/status`, {
+              fetch(`${FRIEND_API_BASE}/voice/stt-segment`, {
                 method: 'POST',
+                body: wavBlob,
+                headers: { 'Content-Type': 'application/octet-stream' },
               })
-              if (!statusRes.ok) return
-              const status = await statusRes.json()
-              if (status.interimText) {
-                onTranscript(status.interimText, false)
-              }
-            } catch {
-              // poll failed, ignore
+                .then(async (res) => {
+                  if (!res.ok) {
+                    const errBody = await res.json().catch(() => ({}))
+                    throw new Error(errBody.error || `STT failed (${res.status})`)
+                  }
+                  const data = await res.json()
+                  if (data.text && data.text.trim()) {
+                    onTranscriptRef.current?.(data.text, true)
+                  }
+                })
+                .catch((err) => {
+                  console.error('[EnergyVAD] STT segment error:', err)
+                  onErrorRef.current?.(err.message)
+                })
             }
-          }, 500)
-          ;(window as any).__friendSttPollId = pollId
-
-          // Schedule first auto-cycle in 3s (for batch STT providers like Groq)
-          cycleTimerRef.current = setTimeout(
-            () => doCycle(onTranscript, myGen),
-            3000,
-          )
-        })
-        .catch((err) => {
-          if (streamingGenRef.current === myGen) {
-            onError(err.message)
           }
-        })
+
+          // Ensure output is connected (needed for Chrome/Chromium)
+          scriptNode.connect(audioCtx.destination)
+          source.connect(scriptNode)
+
+          setConnected(true)
+          console.log('[EnergyVAD] Started')
+        } catch (err: any) {
+          console.error('[EnergyVAD] Init failed:', err)
+          onErrorRef.current?.(String(err))
+        }
+      }
+
+      startCapture()
     },
-    [doCycle],
+    [],
   )
 
-  /**
-   * Stop streaming STT.
-   * Cleans up poll interval and auto-cycle, then returns the final transcript.
-   */
-  const stopStreaming = useCallback(async (): Promise<string> => {
-    // Stop all activity by bumping generation counter
-    streamingGenRef.current++
-    const pollId = (window as any).__friendSttPollId
-    if (pollId) {
-      clearInterval(pollId)
-      delete (window as any).__friendSttPollId
+  function stopCapture(): void {
+    if (scriptNodeRef.current) {
+      try { scriptNodeRef.current.disconnect() } catch {}
+      scriptNodeRef.current = null
     }
-
-    // Stop auto-cycle
-    if (cycleTimerRef.current) {
-      clearTimeout(cycleTimerRef.current)
-      cycleTimerRef.current = null
+    if (sourceNodeRef.current) {
+      try { sourceNodeRef.current.disconnect() } catch {}
+      sourceNodeRef.current = null
     }
-    cycleAccumRef.current = ''
+    if (audioContextRef.current) {
+      try { audioContextRef.current.close() } catch {}
+      audioContextRef.current = null
+    }
+    if (streamRef.current) {
+      streamRef.current.getTracks().forEach(t => t.stop())
+      streamRef.current = null
+    }
+    vadRef.current = null
+  }
 
-    const res = await fetch(`${FRIEND_API_BASE}/voice/stop`, { method: 'POST' })
+  /**
+   * Stop voice call mode.
+   * Cleans up all audio resources.
+   */
+  const stopStreaming = useCallback(async (): Promise<string> => {
+    stopCapture()
     setConnected(false)
-    if (!res.ok) return ''
-    const data = await res.json()
-    return data.text || ''
+    return ''
   }, [])
 
   return {

src/components/friend/frontend/package-lock.json

@@ -11,6 +11,7 @@
         "@pixiv/three-vrm": "^3.4.0",
         "@pixiv/three-vrm-animation": "^3.5.0",
         "@pixiv/three-vrm-core": "^3.5.1",
+        "@tauri-apps/api": "^2.11.1",
         "lucide-react": "^0.577.0",
         "marked": "^18.0.5",
         "mmd-parser": "^1.0.4",
@@ -1358,6 +1359,16 @@
         "win32"
       ]
     },
+    "node_modules/@tauri-apps/api": {
+      "version": "2.11.1",
+      "resolved": "https://registry.npmmirror.com/@tauri-apps/api/-/api-2.11.1.tgz",
+      "integrity": "sha512-M2FPuYND2m+wh5hfW9ZpSdxMPdEJovPBWwoHJmwUpysTYNHaOkVFN419m/K0LIgjb/7KU2vBgsUepJWugQCvAA==",
+      "license": "Apache-2.0 OR MIT",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/tauri"
+      }
+    },
     "node_modules/@tauri-apps/cli": {
       "version": "2.11.3",
       "resolved": "https://registry.npmmirror.com/@tauri-apps/cli/-/cli-2.11.3.tgz",

src/components/friend/frontend/package.json

@@ -11,6 +11,7 @@
     "@pixiv/three-vrm": "^3.4.0",
     "@pixiv/three-vrm-animation": "^3.5.0",
     "@pixiv/three-vrm-core": "^3.5.1",
+    "@tauri-apps/api": "^2.11.1",
     "lucide-react": "^0.577.0",
     "marked": "^18.0.5",
     "mmd-parser": "^1.0.4",

src/components/friend/frontend/src-tauri/Cargo.lock

@@ -4030,6 +4030,7 @@ dependencies = [
  "tauri",
  "tauri-build",
  "tauri-plugin-opener",
+ "webkit2gtk",
 ]
 
 [[package]]

src/components/friend/frontend/src-tauri/Cargo.toml

@@ -17,6 +17,7 @@ tauri = { version = "2", features = [] }
 tauri-plugin-opener = "2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+webkit2gtk = "2.0.2"
 
 [features]
 default = ["custom-protocol"]

src/components/friend/frontend/src-tauri/src/lib.rs

@@ -6,7 +6,22 @@ pub fn run() {
         .plugin(tauri_plugin_opener::init())
         .setup(|app| {
             let window = app.get_webview_window("main").unwrap();
-            window.set_focus().ok();
+
+            // Allow microphone/camera access for browser VAD (getUserMedia)
+            #[cfg(target_os = "linux")]
+            {
+                use webkit2gtk::{PermissionRequestExt, WebViewExt};
+                let _ = window.with_webview(|webview| {
+                    let platform = webview.inner();
+                    platform.connect_permission_request(|_webview, request| {
+                        // Allow all permission requests (media, etc.)
+                        request.allow();
+                        true
+                    });
+                });
+            }
+
+            let _ = window.set_focus();
             Ok(())
         })
         .on_window_event(|window, event| {
@@ -17,4 +32,4 @@ pub fn run() {
 
     builder.run(tauri::generate_context!())
         .expect("error while running tauri application");
-}
+}

src/friend/FriendService.ts

@@ -325,6 +325,14 @@ class FriendService {
    * Stop voice capture and return the accumulated transcript.
    */
   async stopVoiceCapture(): Promise<string> {
+    return this._stopCapture();
+  }
+
+  /**
+   * Internal: stop audio capture + finalize STT, return transcript.
+   */
+  private async _stopCapture(): Promise<string> {
+    console.log(`[FriendService] _stopCapture: capturing=${this.capturing} sttConnection=${this.sttConnection ? 'exists' : 'null'}`);
     if (!this.capturing) return '';
 
     this.capturing = false;
@@ -348,6 +356,7 @@ class FriendService {
     }
 
     const transcript = this.captureTranscripts.join('');
+    console.log(`[FriendService] _stopCapture: transcript="${transcript}" (len=${transcript.length})`);
     this.captureTranscripts = [];
     this.captureInterimText = '';
     this.setState({ captureStatus: { capturing: false } });
@@ -355,10 +364,50 @@ class FriendService {
   }
 
   /**
-   * Get current capture status (for voice call interim polling).
+   * Get current capture status (for push-to-talk mode).
    */
   getCaptureStatus(): { capturing: boolean; interimText?: string } {
-    return this.state.captureStatus ?? { capturing: false };
+    const status = this.state.captureStatus ?? { capturing: false };
+    return { ...status };
+  }
+
+  /**
+   * Transcribe an audio buffer (PCM/WAV) using the configured STT provider
+   * and send the resulting text to the CLI conversation.
+   *
+   * Called by the HTTP endpoint when the browser VAD detects a speech segment.
+   */
+  async transcribeAudioSegment(audioBuffer: Buffer): Promise<string> {
+    const prefs = getPrefs();
+    let provider = prefs.sttProvider;
+    if (!provider || provider === 'browser') {
+      provider = await this.detectAvailableSttProvider();
+    }
+
+    // Create a temporary STT connection just for this segment
+    const conn = await this.startSttConnectionWithTimeout(
+      provider,
+      prefs.sttLanguage || 'zh',
+    );
+
+    try {
+      conn.send(audioBuffer);
+      await conn.finalize();
+      conn.close();
+    } catch (err) {
+      conn.close();
+      throw err;
+    }
+
+    // Get transcript from the finalization callback
+    const transcript = this.captureTranscripts.join('');
+    this.captureTranscripts = [];
+
+    if (transcript.trim()) {
+      this.sendText(transcript);
+    }
+
+    return transcript;
   }
 
   // ── Response broadcast (called by useFriendBridge) ───────────────────
@@ -474,18 +523,54 @@ class FriendService {
 
     this.audioCapture = {
       startRecording: async (onData, _onEnd) => {
-        for (const tool of ['parecord', 'arecord']) {
+        for (const tool of ['arecord', 'parecord']) {
           try {
-            const args = tool === 'parecord'
-              ? ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20']
-              : ['-D', 'default', '-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q'];
+            const args = tool === 'arecord'
+              ? ['-D', 'default', '-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q']
+              : ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20'];
             const proc = spawn(tool, args, { stdio: ['pipe', 'pipe', 'pipe'] });
-            if (proc.pid !== undefined) {
+            if (proc.pid === undefined) continue;
+
+            // Verify the tool actually produces audio data within 500ms.
+            // Some tools (e.g. parecord without PulseAudio) spawn successfully
+            // but exit immediately without any output — catch that here.
+            let dataArrived = false;
+            let verifyTimer: ReturnType<typeof setTimeout> | null = null;
+
+            const verified = await new Promise<boolean>((resolve) => {
+              const dataHandler = (chunk: Buffer) => {
+                dataArrived = true;
+                if (verifyTimer) { clearTimeout(verifyTimer); }
+                // Forward first chunk, then swap to the permanent handler
+                onData(chunk);
+                proc.stdout?.removeListener('data', dataHandler);
+                proc.stdout?.on('data', onData);
+                resolve(true);
+              };
+              proc.stdout?.on('data', dataHandler);
+
+              // Process exited before producing data — mark as failed
+              proc.on('exit', () => {
+                if (!dataArrived) {
+                  if (verifyTimer) { clearTimeout(verifyTimer); }
+                  resolve(false);
+                }
+              });
+
+              // No data within 500ms — assume failure
+              verifyTimer = setTimeout(() => {
+                if (!dataArrived) resolve(false);
+              }, 500);
+            });
+
+            if (verified) {
               captureProc = proc;
-              proc.stdout?.on('data', onData);
               proc.on('exit', () => { captureProc = null; });
               return true;
             }
+
+            // Verification failed — kill and try next tool
+            proc.kill('SIGTERM');
           } catch {
             continue;
           }

src/friend/__tests__/e2e-http-flow.test.ts

@@ -0,0 +1,149 @@
+/**
+ * End-to-end HTTP flow test — simulates what the Tauri frontend does
+ * with the browser VAD + server STT architecture:
+ *   1. POST /voice/stt-segment  → send audio buffer for transcription
+ *   2. Server transcribes and enqueues text to CLI
+ *   3. POST /voice/start + /voice/stop  → push-to-talk mode
+ *
+ * Uses mocked STT so it runs in any environment.
+ */
+import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
+import { friendService } from '../FriendService.js';
+import { setPrefs } from '../prefs.js';
+import { handleFriendApi, setFriendServerInfo } from '../../server/api/friend.js';
+import { getCommandQueue, resetCommandQueue } from '../../utils/messageQueueManager.js';
+
+setFriendServerInfo('127.0.0.1', 3456);
+
+// ════════════════════════════════════════════════════════════════
+// Helpers
+// ════════════════════════════════════════════════════════════════
+
+function makeRequest(method: string, path: string, body?: unknown): Request {
+  const url = new URL(path, 'http://127.0.0.1:3456');
+  const init: RequestInit = { method };
+  if (body) {
+    init.headers = { 'Content-Type': 'application/json' };
+    init.body = JSON.stringify(body);
+  }
+  return new Request(url.toString(), init);
+}
+
+function apiUrl(path: string): URL {
+  return new URL(path, 'http://127.0.0.1:3456');
+}
+
+async function jsonResponse(res: Response): Promise<Record<string, unknown>> {
+  return res.json() as Promise<Record<string, unknown>>;
+}
+
+// ════════════════════════════════════════════════════════════════
+// Mocks
+// ════════════════════════════════════════════════════════════════
+
+function createMockSttFactory() {
+  let counter = 0;
+  return async () => {
+    const myId = ++counter;
+    return {
+      send: () => {},
+      finalize: async () => {
+        await new Promise(r => setTimeout(r, 50));
+        (friendService as any).captureTranscripts.push(`[mock stt result #${myId}]`);
+      },
+      close: () => {},
+    };
+  };
+}
+
+// ════════════════════════════════════════════════════════════════
+// Tests
+// ════════════════════════════════════════════════════════════════
+
+describe('E2E HTTP flow (browser VAD + server STT)', () => {
+  let origStartStt: any;
+
+  beforeEach(() => {
+    resetCommandQueue();
+    setPrefs({ sttProvider: 'groq' });
+    origStartStt = (friendService as any).startSttConnection;
+    (friendService as any).startSttConnection = createMockSttFactory();
+  });
+
+  afterEach(async () => {
+    try { await friendService.stopVoiceCapture(); } catch {}
+    (friendService as any).startSttConnection = origStartStt;
+    resetCommandQueue();
+  });
+
+  test(
+    'POST /voice/stt-segment → transcribes and enqueues via sendText()',
+    async () => {
+      // Create a WAV audio buffer (44-byte header + 16000 samples of silence)
+      const wavHeader = Buffer.alloc(44);
+      wavHeader.write('RIFF', 0);
+      wavHeader.writeUInt32LE(36 + 16000 * 2, 4);
+      wavHeader.write('WAVE', 8);
+      wavHeader.write('fmt ', 12);
+      wavHeader.writeUInt32LE(16, 16);
+      wavHeader.writeUInt16LE(1, 20); // PCM
+      wavHeader.writeUInt16LE(1, 22); // mono
+      wavHeader.writeUInt32LE(16000, 24); // sample rate
+      wavHeader.writeUInt32LE(32000, 28); // byte rate
+      wavHeader.writeUInt16LE(2, 32); // block align
+      wavHeader.writeUInt16LE(16, 34); // bits per sample
+      wavHeader.write('data', 36);
+      wavHeader.writeUInt32LE(16000 * 2, 40);
+
+      const audioData = Buffer.alloc(16000 * 2, 128); // silence
+      const fullAudio = Buffer.concat([wavHeader, audioData]);
+
+      // Send as application/octet-stream
+      const url = new URL('http://127.0.0.1:3456/plugins/friend/voice/stt-segment');
+      const req = new Request(url.toString(), {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/octet-stream' },
+        body: fullAudio,
+      });
+
+      const res = await handleFriendApi(req, url);
+      const data = await jsonResponse(res);
+
+      expect(res.status).toBe(200);
+      expect(data.ok).toBe(true);
+      expect(typeof data.text).toBe('string');
+      expect((data.text as string).length).toBeGreaterThan(0);
+
+      // Verify text was enqueued to CLI via sendText()
+      await new Promise(r => setTimeout(r, 10)); // let async sendText() enqueue
+      const queue = getCommandQueue();
+      expect(queue.length).toBeGreaterThan(0);
+      expect(queue[0].value).toContain('[mock stt result');
+      expect(queue[0].origin?.server).toBe('friend');
+    },
+    10000,
+  );
+
+  test(
+    'POST /voice/start → /voice/stop → push-to-talk cycle',
+    async () => {
+      const startRes = await handleFriendApi(
+        makeRequest('POST', '/plugins/friend/voice/start'),
+        apiUrl('/plugins/friend/voice/start'),
+      );
+      expect(startRes.status).toBe(200);
+
+      // Let it "capture" briefly
+      await new Promise(r => setTimeout(r, 200));
+
+      const stopRes = await handleFriendApi(
+        makeRequest('POST', '/plugins/friend/voice/stop'),
+        apiUrl('/plugins/friend/voice/stop'),
+      );
+      const stopData = await jsonResponse(stopRes);
+      expect(stopRes.status).toBe(200);
+      expect(typeof stopData.text).toBe('string');
+    },
+    15000,
+  );
+});

src/friend/__tests__/voice-cycle.test.ts

@@ -0,0 +1,82 @@
+/**
+ * Voice capture tests for FriendService (push-to-talk mode).
+ *
+ * Tests:
+ *   1. startVoiceCapture → stopVoiceCapture returns transcript
+ *   2. TranscribeAudioSegment sends text to CLI queue
+ */
+import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
+import { friendService } from '../FriendService.js';
+import { setPrefs } from '../prefs.js';
+import { getCommandQueue, resetCommandQueue } from '../../utils/messageQueueManager.js';
+
+function createMockSttConnectionFactory() {
+  let counter = 0;
+  return async () => {
+    const myId = ++counter;
+    return {
+      send: () => {},
+      finalize: async () => {
+        await new Promise(r => setTimeout(r, 50));
+        const fs = friendService as any;
+        fs.captureTranscripts.push(`[mock transcript #${myId}]`);
+      },
+      close: () => {},
+    };
+  };
+}
+
+describe('Voice capture (push-to-talk)', () => {
+  let origStartStt: any;
+  let origAudioCapture: any;
+
+  beforeEach(() => {
+    resetCommandQueue();
+    setPrefs({ sttProvider: 'groq' });
+    origStartStt = (friendService as any).startSttConnection;
+    origAudioCapture = (friendService as any).audioCapture;
+    (friendService as any).startSttConnection = createMockSttConnectionFactory();
+    // Mock audio capture to avoid spawning arecord/parecord
+    (friendService as any).audioCapture = {
+      startRecording: async () => true,
+      stopRecording: async () => {},
+      isRecording: () => false,
+    };
+  });
+
+  afterEach(async () => {
+    try { await friendService.stopVoiceCapture(); } catch {}
+    (friendService as any).startSttConnection = origStartStt;
+    (friendService as any).audioCapture = origAudioCapture;
+    resetCommandQueue();
+  });
+
+  test(
+    'transcribeAudioSegment enqueues text via sendText()',
+    async () => {
+      const audioBuf = Buffer.alloc(16000); // 1s of silence @ 16kHz 16-bit
+      const transcript = await friendService.transcribeAudioSegment(audioBuf);
+
+      expect(transcript).toContain('[mock transcript');
+      await new Promise(r => setTimeout(r, 10)); // let async sendText() enqueue
+      const queue = getCommandQueue();
+      expect(queue.length).toBeGreaterThan(0);
+      expect(queue[0].value).toContain('[mock transcript');
+      expect(queue[0].origin?.server).toBe('friend');
+    },
+    10000,
+  );
+
+  test(
+    'startVoiceCapture + stopVoiceCapture returns transcript',
+    async () => {
+      await friendService.startVoiceCapture();
+
+      await new Promise(r => setTimeout(r, 500));
+
+      const transcript = await friendService.stopVoiceCapture();
+      expect(typeof transcript).toBe('string');
+    },
+    15000,
+  );
+});

src/server/api/friend.ts

@@ -150,7 +150,33 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
     return jsonResponse({ ok: true });
   }
 
-  // ── Voice capture (POST /plugins/friend/voice/start, /stop, /status) ──
+  // ── STT segment from browser VAD (POST /plugins/friend/voice/stt-segment) ──
+  if (pathname === '/plugins/friend/voice/stt-segment' && method === 'POST') {
+    try {
+      const contentType = req.headers.get('content-type') || '';
+      let audioBuffer: Buffer;
+
+      if (contentType.includes('multipart/form-data') || contentType.includes('application/octet-stream')) {
+        const blob = await req.blob();
+        audioBuffer = Buffer.from(await blob.arrayBuffer());
+      } else {
+        // Accept raw PCM or WAV as binary body
+        audioBuffer = Buffer.from(await req.arrayBuffer());
+      }
+
+      if (audioBuffer.length < 100) {
+        return jsonResponse({ error: 'audio too short' }, 400);
+      }
+
+      const transcript = await friendService.transcribeAudioSegment(audioBuffer);
+      return jsonResponse({ ok: true, text: transcript });
+    } catch (err) {
+      console.error('[Friend] STT segment error:', err);
+      return jsonResponse({ error: String(err) }, 500);
+    }
+  }
+
+  // ── Voice capture (POST /plugins/friend/voice/start, /start-vad, /stop, /status) ──
   if (pathname === '/plugins/friend/voice/start' && method === 'POST') {
     try {
       await friendService.startVoiceCapture();
@@ -160,6 +186,15 @@ export async function handleFriendApi(req: Request, url: URL): Promise<Response>
     }
   }
 
+  if (pathname === '/plugins/friend/voice/start-vad' && method === 'POST') {
+    try {
+      await friendService.startVadVoiceCapture();
+      return jsonResponse({ ok: true });
+    } catch (err) {
+      return jsonResponse({ error: String(err) }, 500);
+    }
+  }
+
   if (pathname === '/plugins/friend/voice/stop' && method === 'POST') {
     try {
       const text = await friendService.stopVoiceCapture();

src/server/staticFriend.ts

@@ -25,6 +25,8 @@ const MIME_TYPES: Record<string, string> = {
   '.ico': 'image/x-icon',
   '.woff': 'font/woff',
   '.woff2': 'font/woff2',
+  '.onnx': 'application/octet-stream',
+  '.wasm': 'application/wasm',
 }
 
 export async function handleFriendStaticRequest(req: Request, url: URL): Promise<Response | null> {