fix: Microphone access denied
Browse files
src/components/friend/frontend/components/ChatInput.tsx
CHANGED
|
@@ -170,29 +170,29 @@ export function ChatInput({ visible = true, onActiveChange, uiAlign = 'right', o
|
|
| 170 |
setText('')
|
| 171 |
setOpen(true)
|
| 172 |
|
| 173 |
-
//
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
(
|
| 181 |
-
|
| 182 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
endVoiceCallRef.current()
|
| 184 |
-
}
|
| 185 |
-
}
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
}
|
| 191 |
-
|
| 192 |
-
// Browser STT not available on WebKitGTK β use server STT as fallback
|
| 193 |
-
console.warn('Browser STT not supported, falling back to server STT')
|
| 194 |
-
setVoiceCallActive(false)
|
| 195 |
-
voiceCallActiveRef.current = false
|
| 196 |
}, [handleVoiceCallResult, sttProvider, serverStt, language])
|
| 197 |
|
| 198 |
// --- Voice Call: end ---
|
|
|
|
| 170 |
setText('')
|
| 171 |
setOpen(true)
|
| 172 |
|
| 173 |
+
// Always use server-side STT via HTTP polling.
|
| 174 |
+
// Audio capture is done server-side via cpal (in-process native addon)
|
| 175 |
+
// so no getUserMedia call is needed on the frontend β this avoids
|
| 176 |
+
// WebKitGTK permission issues on Linux.
|
| 177 |
+
serverStt.startStreaming(
|
| 178 |
+
sttProvider,
|
| 179 |
+
(text, isFinal) => {
|
| 180 |
+
handleVoiceCallResult(text, isFinal)
|
| 181 |
+
},
|
| 182 |
+
(err) => {
|
| 183 |
+
console.error('Server STT error:', err)
|
| 184 |
+
if (voiceCallActiveRef.current) {
|
| 185 |
+
// Show error text in the input bar, then end call after 3s
|
| 186 |
+
setText(`θ―ι³ε―ε¨ε€±θ΄₯: ${err}`)
|
| 187 |
+
setTimeout(() => {
|
| 188 |
+
setText('')
|
| 189 |
endVoiceCallRef.current()
|
| 190 |
+
}, 3000)
|
| 191 |
+
}
|
| 192 |
+
},
|
| 193 |
+
language === 'en' ? 'en' : 'zh',
|
| 194 |
+
)
|
| 195 |
+
setRecording(true)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
}, [handleVoiceCallResult, sttProvider, serverStt, language])
|
| 197 |
|
| 198 |
// --- Voice Call: end ---
|
src/components/friend/frontend/hooks/useServerStt.ts
CHANGED
|
@@ -62,8 +62,11 @@ export function useServerStt() {
|
|
| 62 |
) => {
|
| 63 |
// Start capture
|
| 64 |
fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
|
| 65 |
-
.then((res) => {
|
| 66 |
-
if (!res.ok)
|
|
|
|
|
|
|
|
|
|
| 67 |
setConnected(true)
|
| 68 |
|
| 69 |
// Poll for interim results every 500ms
|
|
|
|
| 62 |
) => {
|
| 63 |
// Start capture
|
| 64 |
fetch(`${FRIEND_API_BASE}/voice/start`, { method: 'POST' })
|
| 65 |
+
.then(async (res) => {
|
| 66 |
+
if (!res.ok) {
|
| 67 |
+
const body = await res.json().catch(() => ({}))
|
| 68 |
+
throw new Error(body.error || 'STT start failed')
|
| 69 |
+
}
|
| 70 |
setConnected(true)
|
| 71 |
|
| 72 |
// Poll for interim results every 500ms
|
src/friend/FriendService.ts
CHANGED
|
@@ -154,42 +154,33 @@ class FriendService {
|
|
| 154 |
/**
|
| 155 |
* Start in-process voice capture using cpal.
|
| 156 |
* Audio is forwarded to the configured STT provider.
|
| 157 |
-
*
|
|
|
|
| 158 |
*/
|
| 159 |
async startVoiceCapture(): Promise<void> {
|
| 160 |
if (this.capturing) return;
|
| 161 |
|
| 162 |
const prefs = getPrefs();
|
| 163 |
-
|
| 164 |
const language = prefs.sttLanguage || 'zh';
|
| 165 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
this.captureTranscripts = [];
|
| 167 |
this.captureInterimText = '';
|
| 168 |
this.capturing = true;
|
| 169 |
|
| 170 |
try {
|
| 171 |
-
//
|
| 172 |
-
|
| 173 |
-
this.
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
// 3. Start cpal recording β chunks go to STT
|
| 179 |
-
const ok = await audio.startRecording(
|
| 180 |
-
(chunk: Buffer) => {
|
| 181 |
-
this.sttConnection?.send(chunk);
|
| 182 |
-
},
|
| 183 |
-
() => {
|
| 184 |
-
// Capture ended (user stop or silence detection)
|
| 185 |
-
},
|
| 186 |
);
|
| 187 |
-
|
| 188 |
-
if (!ok) {
|
| 189 |
-
// Fallback: try arecord as subprocess
|
| 190 |
-
this.capturing = false;
|
| 191 |
-
throw new Error('Native audio capture unavailable');
|
| 192 |
-
}
|
| 193 |
} catch (err) {
|
| 194 |
this.capturing = false;
|
| 195 |
this.sttConnection?.close();
|
|
@@ -198,6 +189,128 @@ class FriendService {
|
|
| 198 |
}
|
| 199 |
}
|
| 200 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
/**
|
| 202 |
* Stop voice capture and return the accumulated transcript.
|
| 203 |
*/
|
|
@@ -329,40 +442,15 @@ class FriendService {
|
|
| 329 |
}
|
| 330 |
}
|
| 331 |
|
| 332 |
-
// ββ Private: Audio capture (
|
| 333 |
|
| 334 |
private async loadAudioCapture(): Promise<AudioCaptureProvider> {
|
| 335 |
if (this.audioCapture) return this.audioCapture;
|
| 336 |
|
| 337 |
-
//
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
this.audioCapture = {
|
| 342 |
-
startRecording: async (onData, onEnd) => {
|
| 343 |
-
try {
|
| 344 |
-
return mod.startNativeRecording(
|
| 345 |
-
(data: Buffer) => onData(data),
|
| 346 |
-
() => onEnd(),
|
| 347 |
-
) as boolean;
|
| 348 |
-
} catch {
|
| 349 |
-
return false;
|
| 350 |
-
}
|
| 351 |
-
},
|
| 352 |
-
stopRecording: async () => {
|
| 353 |
-
if (mod.isNativeRecordingActive()) {
|
| 354 |
-
mod.stopNativeRecording();
|
| 355 |
-
}
|
| 356 |
-
},
|
| 357 |
-
isRecording: () => mod.isNativeRecordingActive() as boolean,
|
| 358 |
-
};
|
| 359 |
-
return this.audioCapture;
|
| 360 |
-
}
|
| 361 |
-
} catch {
|
| 362 |
-
// cpal unavailable, fall through
|
| 363 |
-
}
|
| 364 |
-
|
| 365 |
-
// Fallback: spawn arecord/parecord as subprocess
|
| 366 |
const { spawn } = await import('node:child_process');
|
| 367 |
let captureProc: import('node:child_process').ChildProcess | null = null;
|
| 368 |
|
|
@@ -372,7 +460,7 @@ class FriendService {
|
|
| 372 |
try {
|
| 373 |
const args = tool === 'parecord'
|
| 374 |
? ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20']
|
| 375 |
-
: ['-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q'
|
| 376 |
const proc = spawn(tool, args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
| 377 |
if (proc.pid !== undefined) {
|
| 378 |
captureProc = proc;
|
|
|
|
| 154 |
/**
|
| 155 |
* Start in-process voice capture using cpal.
|
| 156 |
* Audio is forwarded to the configured STT provider.
|
| 157 |
+
* Wraps initialization in a timeout (12s) to prevent hanging
|
| 158 |
+
* when STT provider or audio device is unavailable.
|
| 159 |
*/
|
| 160 |
async startVoiceCapture(): Promise<void> {
|
| 161 |
if (this.capturing) return;
|
| 162 |
|
| 163 |
const prefs = getPrefs();
|
| 164 |
+
let provider = prefs.sttProvider;
|
| 165 |
const language = prefs.sttLanguage || 'zh';
|
| 166 |
|
| 167 |
+
// Auto-detect STT provider if not configured or set to 'browser' (not available in WebKitGTK)
|
| 168 |
+
if (!provider || provider === 'browser') {
|
| 169 |
+
provider = await this.detectAvailableSttProvider();
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
this.captureTranscripts = [];
|
| 173 |
this.captureInterimText = '';
|
| 174 |
this.capturing = true;
|
| 175 |
|
| 176 |
try {
|
| 177 |
+
// Wrap the whole initialization in a 12s timeout to avoid hanging
|
| 178 |
+
// when STT provider or audio device is unavailable.
|
| 179 |
+
await this.withTimeout(
|
| 180 |
+
this._initVoiceCapture(provider, language),
|
| 181 |
+
12000,
|
| 182 |
+
`Voice initialization timed out. Check that your microphone is accessible and STT provider "${provider}" is configured correctly.`,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
} catch (err) {
|
| 185 |
this.capturing = false;
|
| 186 |
this.sttConnection?.close();
|
|
|
|
| 189 |
}
|
| 190 |
}
|
| 191 |
|
| 192 |
+
/**
|
| 193 |
+
* Internal voice capture initialization (STT connection + audio capture).
|
| 194 |
+
* Separated so startVoiceCapture() can wrap it with a timeout.
|
| 195 |
+
*/
|
| 196 |
+
private async _initVoiceCapture(
|
| 197 |
+
provider: string,
|
| 198 |
+
language: string,
|
| 199 |
+
): Promise<void> {
|
| 200 |
+
// 1. Start STT provider connection (with inner 8s timeout)
|
| 201 |
+
const conn = await this.startSttConnectionWithTimeout(provider, language);
|
| 202 |
+
this.sttConnection = conn;
|
| 203 |
+
|
| 204 |
+
// 2. Load audio capture module (cpal)
|
| 205 |
+
const audio = await this.loadAudioCapture();
|
| 206 |
+
|
| 207 |
+
// 3. Start cpal recording β chunks go to STT
|
| 208 |
+
const ok = await audio.startRecording(
|
| 209 |
+
(chunk: Buffer) => {
|
| 210 |
+
this.sttConnection?.send(chunk);
|
| 211 |
+
},
|
| 212 |
+
() => {
|
| 213 |
+
// Capture ended (user stop or silence detection)
|
| 214 |
+
},
|
| 215 |
+
);
|
| 216 |
+
|
| 217 |
+
if (!ok) {
|
| 218 |
+
throw new Error('Native audio capture unavailable');
|
| 219 |
+
}
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
/** Race a promise against a timeout */
|
| 223 |
+
private async withTimeout<T>(
|
| 224 |
+
promise: Promise<T>,
|
| 225 |
+
ms: number,
|
| 226 |
+
message: string,
|
| 227 |
+
): Promise<T> {
|
| 228 |
+
return Promise.race([
|
| 229 |
+
promise,
|
| 230 |
+
new Promise<never>((_, reject) =>
|
| 231 |
+
setTimeout(() => reject(new Error(message)), ms),
|
| 232 |
+
),
|
| 233 |
+
]);
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
/**
|
| 237 |
+
* Auto-detect the first available STT provider.
|
| 238 |
+
* Tries: local Whisper β Anthropic Voice Stream β Doubao ASR
|
| 239 |
+
*/
|
| 240 |
+
private async detectAvailableSttProvider(): Promise<string> {
|
| 241 |
+
console.log('[FriendService] detectAvailableSttProvider: checking available providers...');
|
| 242 |
+
|
| 243 |
+
// Check local Whisper first (no external API keys needed)
|
| 244 |
+
try {
|
| 245 |
+
const { checkLocalWhisperAvailable } = await import(
|
| 246 |
+
'../services/voice/whisperSTT.js'
|
| 247 |
+
);
|
| 248 |
+
const avail = await checkLocalWhisperAvailable();
|
| 249 |
+
console.log('[FriendService] detectAvailableSttProvider: local Whisper available:', avail);
|
| 250 |
+
if (avail) {
|
| 251 |
+
return 'local';
|
| 252 |
+
}
|
| 253 |
+
} catch (e) {
|
| 254 |
+
console.warn('[FriendService] detectAvailableSttProvider: local whisper check failed:', e);
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
// Check Anthropic Voice Stream
|
| 258 |
+
try {
|
| 259 |
+
const { isVoiceStreamAvailable } = await import(
|
| 260 |
+
'../services/voiceStreamSTT.js'
|
| 261 |
+
);
|
| 262 |
+
if (isVoiceStreamAvailable()) {
|
| 263 |
+
return 'anthropic';
|
| 264 |
+
}
|
| 265 |
+
} catch { /* skip */ }
|
| 266 |
+
|
| 267 |
+
// Check Doubao credentials file
|
| 268 |
+
try {
|
| 269 |
+
const path = await import('node:path');
|
| 270 |
+
const fs = await import('node:fs');
|
| 271 |
+
const homeDir = process.env.HOME || process.env.USERPROFILE || '';
|
| 272 |
+
const credsPath = path.join(homeDir, '.claude', 'tts', 'doubao', 'credentials.json');
|
| 273 |
+
if (fs.existsSync(credsPath)) {
|
| 274 |
+
return 'doubao';
|
| 275 |
+
}
|
| 276 |
+
} catch { /* skip */ }
|
| 277 |
+
|
| 278 |
+
throw new Error(
|
| 279 |
+
'No STT provider available. Install local Whisper:\n' +
|
| 280 |
+
' pip install openai-whisper\n\n' +
|
| 281 |
+
'Or configure an STT provider in Friend settings (Settings β STT Provider).',
|
| 282 |
+
);
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
/**
|
| 286 |
+
* Start STT connection with a timeout to prevent hanging
|
| 287 |
+
* when the provider is unavailable (e.g. Python Whisper not installed).
|
| 288 |
+
*/
|
| 289 |
+
private async startSttConnectionWithTimeout(
|
| 290 |
+
provider: string,
|
| 291 |
+
language: string,
|
| 292 |
+
): Promise<{ send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void }> {
|
| 293 |
+
const timeoutMs = 8000;
|
| 294 |
+
const result = await Promise.race([
|
| 295 |
+
this.startSttConnection(provider, language),
|
| 296 |
+
new Promise<never>((_, reject) =>
|
| 297 |
+
setTimeout(
|
| 298 |
+
() =>
|
| 299 |
+
reject(
|
| 300 |
+
new Error(
|
| 301 |
+
`STT provider "${provider}" timed out after ${timeoutMs / 1000}s.` +
|
| 302 |
+
(provider === 'local'
|
| 303 |
+
? '\nInstall local Whisper: pip install openai-whisper'
|
| 304 |
+
: ''),
|
| 305 |
+
),
|
| 306 |
+
),
|
| 307 |
+
timeoutMs,
|
| 308 |
+
),
|
| 309 |
+
),
|
| 310 |
+
]);
|
| 311 |
+
return result;
|
| 312 |
+
}
|
| 313 |
+
|
| 314 |
/**
|
| 315 |
* Stop voice capture and return the accumulated transcript.
|
| 316 |
*/
|
|
|
|
| 442 |
}
|
| 443 |
}
|
| 444 |
|
| 445 |
+
// ββ Private: Audio capture (arecord/parecord subprocess) ββββββββββββββ
|
| 446 |
|
| 447 |
private async loadAudioCapture(): Promise<AudioCaptureProvider> {
|
| 448 |
if (this.audioCapture) return this.audioCapture;
|
| 449 |
|
| 450 |
+
// Use subprocess-based capture (arecord/parecord) on all platforms.
|
| 451 |
+
// Skipping the native cpal module because its synchronous NAPI call
|
| 452 |
+
// can block the event loop if ALSA initialization hangs, and there
|
| 453 |
+
// is no way to timeout a native binding call from JS.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 454 |
const { spawn } = await import('node:child_process');
|
| 455 |
let captureProc: import('node:child_process').ChildProcess | null = null;
|
| 456 |
|
|
|
|
| 460 |
try {
|
| 461 |
const args = tool === 'parecord'
|
| 462 |
? ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20']
|
| 463 |
+
: ['-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q'];
|
| 464 |
const proc = spawn(tool, args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
| 465 |
if (proc.pid !== undefined) {
|
| 466 |
captureProc = proc;
|
src/friend/server.ts
CHANGED
|
@@ -86,6 +86,7 @@ export function startFriendServer(port = 3456, host = '127.0.0.1'): ReturnType<t
|
|
| 86 |
server = Bun.serve<undefined>({
|
| 87 |
port,
|
| 88 |
hostname: host,
|
|
|
|
| 89 |
async fetch(req) {
|
| 90 |
const url = new URL(req.url);
|
| 91 |
|
|
|
|
| 86 |
server = Bun.serve<undefined>({
|
| 87 |
port,
|
| 88 |
hostname: host,
|
| 89 |
+
idleTimeout: 60, // seconds β allow slow STT provider init (Python/PyTorch imports)
|
| 90 |
async fetch(req) {
|
| 91 |
const url = new URL(req.url);
|
| 92 |
|
src/services/voice/whisperSTT.ts
CHANGED
|
@@ -217,16 +217,23 @@ export function connectLocalWhisperStream(
|
|
| 217 |
})
|
| 218 |
}
|
| 219 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
export async function checkLocalWhisperAvailable(): Promise<boolean> {
|
| 221 |
try {
|
| 222 |
const python = resolvePythonPath()
|
| 223 |
return await new Promise(resolve => {
|
| 224 |
-
const proc = spawn(
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
|
|
|
|
|
|
|
|
|
| 230 |
})
|
| 231 |
} catch {
|
| 232 |
return false
|
|
|
|
| 217 |
})
|
| 218 |
}
|
| 219 |
|
| 220 |
+
/**
|
| 221 |
+
* Fast check for local whisper availability using `find_spec` (no actual
|
| 222 |
+
* module import, avoiding the slow PyTorch/numpy import chain).
|
| 223 |
+
*/
|
| 224 |
export async function checkLocalWhisperAvailable(): Promise<boolean> {
|
| 225 |
try {
|
| 226 |
const python = resolvePythonPath()
|
| 227 |
return await new Promise(resolve => {
|
| 228 |
+
const proc = spawn(
|
| 229 |
+
python,
|
| 230 |
+
[
|
| 231 |
+
'-c',
|
| 232 |
+
'import importlib.util,sys; sys.exit(0 if importlib.util.find_spec("whisper") else 1)',
|
| 233 |
+
],
|
| 234 |
+
{ stdio: ['ignore', 'pipe', 'pipe'] },
|
| 235 |
+
)
|
| 236 |
+
proc.on('close', code => resolve(code === 0))
|
| 237 |
})
|
| 238 |
} catch {
|
| 239 |
return false
|