chenbhao commited on
Commit
bc9c65c
Β·
1 Parent(s): d530cd8

fix: Microphone access denied

Browse files
src/components/friend/frontend/components/ChatInput.tsx CHANGED
@@ -170,29 +170,29 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
170
  setText('')
171
  setOpen(true)
172
 
173
- // Server STT streaming via HTTP polling
174
- if (sttProvider !== 'browser') {
175
- serverStt.startStreaming(
176
- sttProvider,
177
- (text, isFinal) => {
178
- handleVoiceCallResult(text, isFinal)
179
- },
180
- (err) => {
181
- console.error('Server STT error:', err)
182
- if (voiceCallActiveRef.current) {
 
 
 
 
 
 
183
  endVoiceCallRef.current()
184
- }
185
- },
186
- language === 'en' ? 'en' : 'zh',
187
- )
188
- setRecording(true)
189
- return
190
- }
191
-
192
- // Browser STT not available on WebKitGTK β€” use server STT as fallback
193
- console.warn('Browser STT not supported, falling back to server STT')
194
- setVoiceCallActive(false)
195
- voiceCallActiveRef.current = false
196
  }, [handleVoiceCallResult, sttProvider, serverStt, language])
197
 
198
  // --- Voice Call: end ---
 
170
  setText('')
171
  setOpen(true)
172
 
173
+ // Always use server-side STT via HTTP polling.
174
+ // Audio capture is done server-side via cpal (in-process native addon)
175
+ // so no getUserMedia call is needed on the frontend β€” this avoids
176
+ // WebKitGTK permission issues on Linux.
177
+ serverStt.startStreaming(
178
+ sttProvider,
179
+ (text, isFinal) => {
180
+ handleVoiceCallResult(text, isFinal)
181
+ },
182
+ (err) => {
183
+ console.error('Server STT error:', err)
184
+ if (voiceCallActiveRef.current) {
185
+ // Show error text in the input bar, then end call after 3s
186
+ setText(`语音启动倱θ΄₯: ${err}`)
187
+ setTimeout(() => {
188
+ setText('')
189
  endVoiceCallRef.current()
190
+ }, 3000)
191
+ }
192
+ },
193
+ language === 'en' ? 'en' : 'zh',
194
+ )
195
+ setRecording(true)
 
 
 
 
 
 
196
  }, [handleVoiceCallResult, sttProvider, serverStt, language])
197
 
198
  // --- Voice Call: end ---
src/components/friend/frontend/hooks/useServerStt.ts CHANGED
@@ -62,8 +62,11 @@ export function useServerStt() {
62
  ) => {
63
  // Start capture
64
  fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
65
- .then((res) => {
66
- if (!res.ok) throw new Error('STT start failed')
 
 
 
67
  setConnected(true)
68
 
69
  // Poll for interim results every 500ms
 
62
  ) => {
63
  // Start capture
64
  fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
65
+ .then(async (res) => {
66
+ if (!res.ok) {
67
+ const body = await res.json().catch(() => ({}))
68
+ throw new Error(body.error || 'STT start failed')
69
+ }
70
  setConnected(true)
71
 
72
  // Poll for interim results every 500ms
src/friend/FriendService.ts CHANGED
@@ -154,42 +154,33 @@ class FriendService {
154
  /**
155
  * Start in-process voice capture using cpal.
156
  * Audio is forwarded to the configured STT provider.
157
- * Returns when capture-started confirmation is received.
 
158
  */
159
  async startVoiceCapture(): Promise<void> {
160
  if (this.capturing) return;
161
 
162
  const prefs = getPrefs();
163
- const provider = prefs.sttProvider || 'browser';
164
  const language = prefs.sttLanguage || 'zh';
165
 
 
 
 
 
 
166
  this.captureTranscripts = [];
167
  this.captureInterimText = '';
168
  this.capturing = true;
169
 
170
  try {
171
- // 1. Start STT provider connection
172
- const conn = await this.startSttConnection(provider, language);
173
- this.sttConnection = conn;
174
-
175
- // 2. Load audio capture module (cpal)
176
- const audio = await this.loadAudioCapture();
177
-
178
- // 3. Start cpal recording β€” chunks go to STT
179
- const ok = await audio.startRecording(
180
- (chunk: Buffer) => {
181
- this.sttConnection?.send(chunk);
182
- },
183
- () => {
184
- // Capture ended (user stop or silence detection)
185
- },
186
  );
187
-
188
- if (!ok) {
189
- // Fallback: try arecord as subprocess
190
- this.capturing = false;
191
- throw new Error('Native audio capture unavailable');
192
- }
193
  } catch (err) {
194
  this.capturing = false;
195
  this.sttConnection?.close();
@@ -198,6 +189,128 @@ class FriendService {
198
  }
199
  }
200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
201
  /**
202
  * Stop voice capture and return the accumulated transcript.
203
  */
@@ -329,40 +442,15 @@ class FriendService {
329
  }
330
  }
331
 
332
- // ── Private: Audio capture (cpal wrapper) ────────────────────────────
333
 
334
  private async loadAudioCapture(): Promise<AudioCaptureProvider> {
335
  if (this.audioCapture) return this.audioCapture;
336
 
337
- // Try native cpal module first
338
- try {
339
- const mod = await import('audio-capture-napi').catch(() => null);
340
- if (mod && typeof mod.startNativeRecording === 'function') {
341
- this.audioCapture = {
342
- startRecording: async (onData, onEnd) => {
343
- try {
344
- return mod.startNativeRecording(
345
- (data: Buffer) => onData(data),
346
- () => onEnd(),
347
- ) as boolean;
348
- } catch {
349
- return false;
350
- }
351
- },
352
- stopRecording: async () => {
353
- if (mod.isNativeRecordingActive()) {
354
- mod.stopNativeRecording();
355
- }
356
- },
357
- isRecording: () => mod.isNativeRecordingActive() as boolean,
358
- };
359
- return this.audioCapture;
360
- }
361
- } catch {
362
- // cpal unavailable, fall through
363
- }
364
-
365
- // Fallback: spawn arecord/parecord as subprocess
366
  const { spawn } = await import('node:child_process');
367
  let captureProc: import('node:child_process').ChildProcess | null = null;
368
 
@@ -372,7 +460,7 @@ class FriendService {
372
  try {
373
  const args = tool === 'parecord'
374
  ? ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20']
375
- : ['-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q', '-'];
376
  const proc = spawn(tool, args, { stdio: ['pipe', 'pipe', 'pipe'] });
377
  if (proc.pid !== undefined) {
378
  captureProc = proc;
 
154
  /**
155
  * Start in-process voice capture using cpal.
156
  * Audio is forwarded to the configured STT provider.
157
+ * Wraps initialization in a timeout (12s) to prevent hanging
158
+ * when STT provider or audio device is unavailable.
159
  */
160
  async startVoiceCapture(): Promise<void> {
161
  if (this.capturing) return;
162
 
163
  const prefs = getPrefs();
164
+ let provider = prefs.sttProvider;
165
  const language = prefs.sttLanguage || 'zh';
166
 
167
+ // Auto-detect STT provider if not configured or set to 'browser' (not available in WebKitGTK)
168
+ if (!provider || provider === 'browser') {
169
+ provider = await this.detectAvailableSttProvider();
170
+ }
171
+
172
  this.captureTranscripts = [];
173
  this.captureInterimText = '';
174
  this.capturing = true;
175
 
176
  try {
177
+ // Wrap the whole initialization in a 12s timeout to avoid hanging
178
+ // when STT provider or audio device is unavailable.
179
+ await this.withTimeout(
180
+ this._initVoiceCapture(provider, language),
181
+ 12000,
182
+ `Voice initialization timed out. Check that your microphone is accessible and STT provider "${provider}" is configured correctly.`,
 
 
 
 
 
 
 
 
 
183
  );
 
 
 
 
 
 
184
  } catch (err) {
185
  this.capturing = false;
186
  this.sttConnection?.close();
 
189
  }
190
  }
191
 
192
+ /**
193
+ * Internal voice capture initialization (STT connection + audio capture).
194
+ * Separated so startVoiceCapture() can wrap it with a timeout.
195
+ */
196
+ private async _initVoiceCapture(
197
+ provider: string,
198
+ language: string,
199
+ ): Promise<void> {
200
+ // 1. Start STT provider connection (with inner 8s timeout)
201
+ const conn = await this.startSttConnectionWithTimeout(provider, language);
202
+ this.sttConnection = conn;
203
+
204
+ // 2. Load audio capture module (cpal)
205
+ const audio = await this.loadAudioCapture();
206
+
207
+ // 3. Start cpal recording β€” chunks go to STT
208
+ const ok = await audio.startRecording(
209
+ (chunk: Buffer) => {
210
+ this.sttConnection?.send(chunk);
211
+ },
212
+ () => {
213
+ // Capture ended (user stop or silence detection)
214
+ },
215
+ );
216
+
217
+ if (!ok) {
218
+ throw new Error('Native audio capture unavailable');
219
+ }
220
+ }
221
+
222
+ /** Race a promise against a timeout */
223
+ private async withTimeout<T>(
224
+ promise: Promise<T>,
225
+ ms: number,
226
+ message: string,
227
+ ): Promise<T> {
228
+ return Promise.race([
229
+ promise,
230
+ new Promise<never>((_, reject) =>
231
+ setTimeout(() => reject(new Error(message)), ms),
232
+ ),
233
+ ]);
234
+ }
235
+
236
+ /**
237
+ * Auto-detect the first available STT provider.
238
+ * Tries: local Whisper β†’ Anthropic Voice Stream β†’ Doubao ASR
239
+ */
240
+ private async detectAvailableSttProvider(): Promise<string> {
241
+ console.log('[FriendService] detectAvailableSttProvider: checking available providers...');
242
+
243
+ // Check local Whisper first (no external API keys needed)
244
+ try {
245
+ const { checkLocalWhisperAvailable } = await import(
246
+ '../services/voice/whisperSTT.js'
247
+ );
248
+ const avail = await checkLocalWhisperAvailable();
249
+ console.log('[FriendService] detectAvailableSttProvider: local Whisper available:', avail);
250
+ if (avail) {
251
+ return 'local';
252
+ }
253
+ } catch (e) {
254
+ console.warn('[FriendService] detectAvailableSttProvider: local whisper check failed:', e);
255
+ }
256
+
257
+ // Check Anthropic Voice Stream
258
+ try {
259
+ const { isVoiceStreamAvailable } = await import(
260
+ '../services/voiceStreamSTT.js'
261
+ );
262
+ if (isVoiceStreamAvailable()) {
263
+ return 'anthropic';
264
+ }
265
+ } catch { /* skip */ }
266
+
267
+ // Check Doubao credentials file
268
+ try {
269
+ const path = await import('node:path');
270
+ const fs = await import('node:fs');
271
+ const homeDir = process.env.HOME || process.env.USERPROFILE || '';
272
+ const credsPath = path.join(homeDir, '.claude', 'tts', 'doubao', 'credentials.json');
273
+ if (fs.existsSync(credsPath)) {
274
+ return 'doubao';
275
+ }
276
+ } catch { /* skip */ }
277
+
278
+ throw new Error(
279
+ 'No STT provider available. Install local Whisper:\n' +
280
+ ' pip install openai-whisper\n\n' +
281
+ 'Or configure an STT provider in Friend settings (Settings β†’ STT Provider).',
282
+ );
283
+ }
284
+
285
+ /**
286
+ * Start STT connection with a timeout to prevent hanging
287
+ * when the provider is unavailable (e.g. Python Whisper not installed).
288
+ */
289
+ private async startSttConnectionWithTimeout(
290
+ provider: string,
291
+ language: string,
292
+ ): Promise<{ send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void }> {
293
+ const timeoutMs = 8000;
294
+ const result = await Promise.race([
295
+ this.startSttConnection(provider, language),
296
+ new Promise<never>((_, reject) =>
297
+ setTimeout(
298
+ () =>
299
+ reject(
300
+ new Error(
301
+ `STT provider "${provider}" timed out after ${timeoutMs / 1000}s.` +
302
+ (provider === 'local'
303
+ ? '\nInstall local Whisper: pip install openai-whisper'
304
+ : ''),
305
+ ),
306
+ ),
307
+ timeoutMs,
308
+ ),
309
+ ),
310
+ ]);
311
+ return result;
312
+ }
313
+
314
  /**
315
  * Stop voice capture and return the accumulated transcript.
316
  */
 
442
  }
443
  }
444
 
445
+ // ── Private: Audio capture (arecord/parecord subprocess) ──────────────
446
 
447
  private async loadAudioCapture(): Promise<AudioCaptureProvider> {
448
  if (this.audioCapture) return this.audioCapture;
449
 
450
+ // Use subprocess-based capture (arecord/parecord) on all platforms.
451
+ // Skipping the native cpal module because its synchronous NAPI call
452
+ // can block the event loop if ALSA initialization hangs, and there
453
+ // is no way to timeout a native binding call from JS.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
454
  const { spawn } = await import('node:child_process');
455
  let captureProc: import('node:child_process').ChildProcess | null = null;
456
 
 
460
  try {
461
  const args = tool === 'parecord'
462
  ? ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20']
463
+ : ['-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q'];
464
  const proc = spawn(tool, args, { stdio: ['pipe', 'pipe', 'pipe'] });
465
  if (proc.pid !== undefined) {
466
  captureProc = proc;
src/friend/server.ts CHANGED
@@ -86,6 +86,7 @@ export function startFriendServer(port = 3456, host = '127.0.0.1'): ReturnType<t
86
  server = Bun.serve<undefined>({
87
  port,
88
  hostname: host,
 
89
  async fetch(req) {
90
  const url = new URL(req.url);
91
 
 
86
  server = Bun.serve<undefined>({
87
  port,
88
  hostname: host,
89
+ idleTimeout: 60, // seconds β€” allow slow STT provider init (Python/PyTorch imports)
90
  async fetch(req) {
91
  const url = new URL(req.url);
92
 
src/services/voice/whisperSTT.ts CHANGED
@@ -217,16 +217,23 @@ export function connectLocalWhisperStream(
217
  })
218
  }
219
 
 
 
 
 
220
  export async function checkLocalWhisperAvailable(): Promise<boolean> {
221
  try {
222
  const python = resolvePythonPath()
223
  return await new Promise(resolve => {
224
- const proc = spawn(python, ['-c', 'import whisper; print("ok")'], {
225
- stdio: ['ignore', 'pipe', 'pipe'],
226
- })
227
- let out = ''
228
- proc.stdout.on('data', d => { out += d.toString() })
229
- proc.on('close', () => { resolve(out.trim() === 'ok') })
 
 
 
230
  })
231
  } catch {
232
  return false
 
217
  })
218
  }
219
 
220
+ /**
221
+ * Fast check for local whisper availability using `find_spec` (no actual
222
+ * module import, avoiding the slow PyTorch/numpy import chain).
223
+ */
224
  export async function checkLocalWhisperAvailable(): Promise<boolean> {
225
  try {
226
  const python = resolvePythonPath()
227
  return await new Promise(resolve => {
228
+ const proc = spawn(
229
+ python,
230
+ [
231
+ '-c',
232
+ 'import importlib.util,sys; sys.exit(0 if importlib.util.find_spec("whisper") else 1)',
233
+ ],
234
+ { stdio: ['ignore', 'pipe', 'pipe'] },
235
+ )
236
+ proc.on('close', code => resolve(code === 0))
237
  })
238
  } catch {
239
  return false