fix: audio transmission

wait for audio synthesis to complete before playback instead of chunking

desktop/src/hooks/useCompanionAudio.ts

@@ -1,48 +1,57 @@
 import { useRef, useCallback, useEffect } from 'react'
 
-function pcmToWav(pcm: ArrayBuffer): Blob {
-  const int16 = new Int16Array(pcm)
-  const numSamples = int16.length
+/** PCM Int16 → WAV Blob */
+function pcmChunksToWav(chunks: Int16Array[]): Blob {
+  if (chunks.length === 0) {
+    // Return valid empty WAV
+    const empty = new ArrayBuffer(44)
+    const v = new DataView(empty)
+    const w = (o: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)) }
+    w(0, 'RIFF'); v.setUint32(4, 36, true); w(8, 'WAVE')
+    w(12, 'fmt '); v.setUint32(16, 16, true); v.setUint16(20, 1, true)
+    v.setUint16(22, 1, true); v.setUint32(24, 24000, true); v.setUint32(28, 48000, true)
+    v.setUint16(32, 2, true); v.setUint16(34, 16, true)
+    w(36, 'data'); v.setUint32(40, 0, true)
+    return new Blob([empty], { type: 'audio/wav' })
+  }
+
+  const totalSamples = chunks.reduce((s, c) => s + c.length, 0)
   const sampleRate = 24000
-  const numChannels = 1
   const bitsPerSample = 16
-  const byteRate = (sampleRate * numChannels * bitsPerSample) / 8
+  const numChannels = 1
   const blockAlign = (numChannels * bitsPerSample) / 8
-  const dataSize = numSamples * blockAlign
-  const headerSize = 44
-  const totalSize = headerSize + dataSize
+  const byteRate = sampleRate * blockAlign
+  const dataSize = totalSamples * blockAlign
+  const totalSize = 44 + dataSize
 
   const buf = new ArrayBuffer(totalSize)
-  const view = new DataView(buf)
-
-  const w = (offset: number, str: string) => {
-    for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i))
+  const v = new DataView(buf)
+  const w = (o: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)) }
+
+  w(0, 'RIFF'); v.setUint32(4, totalSize - 8, true); w(8, 'WAVE')
+  w(12, 'fmt '); v.setUint32(16, 16, true); v.setUint16(20, 1, true)
+  v.setUint16(22, numChannels, true); v.setUint32(24, sampleRate, true)
+  v.setUint32(28, byteRate, true); v.setUint16(32, blockAlign, true)
+  v.setUint16(34, bitsPerSample, true)
+  w(36, 'data'); v.setUint32(40, dataSize, true)
+
+  const pcmView = new Int16Array(buf, 44, totalSamples)
+  let offset = 0
+  for (const chunk of chunks) {
+    pcmView.set(chunk, offset)
+    offset += chunk.length
   }
-
-  w(0, 'RIFF')
-  view.setUint32(4, totalSize - 8, true)
-  w(8, 'WAVE')
-  w(12, 'fmt ')
-  view.setUint32(16, 16, true)
-  view.setUint16(20, 1, true) // PCM
-  view.setUint16(22, numChannels, true)
-  view.setUint32(24, sampleRate, true)
-  view.setUint32(28, byteRate, true)
-  view.setUint16(32, blockAlign, true)
-  view.setUint16(34, bitsPerSample, true)
-  w(36, 'data')
-  view.setUint32(40, dataSize, true)
-
-  new Int16Array(buf, headerSize, numSamples).set(int16)
   return new Blob([buf], { type: 'audio/wav' })
 }
 
 export function useCompanionAudio() {
   const audioRef = useRef<HTMLAudioElement | null>(null)
-  const queueRef = useRef<Blob[]>([])
-  const playingRef = useRef(false)
   const urlRef = useRef<string | null>(null)
 
+  // Accumulation buffer — all PCM chunks are poured here until flush()
+  const chunksRef = useRef<Int16Array[]>([])
+  const chunksLenRef = useRef(0) // track total sample count for quick lookup
+
   const revokeUrl = useCallback(() => {
     if (urlRef.current) {
       URL.revokeObjectURL(urlRef.current)
@@ -50,43 +59,26 @@ export function useCompanionAudio() {
     }
   }, [])
 
-  const playNext = useCallback(() => {
-    if (playingRef.current) return
-    const queue = queueRef.current
-    if (queue.length === 0) return
-
-    const blob = queue.shift()!
-    const url = URL.createObjectURL(blob)
-    const audio = audioRef.current
-    if (!audio) return
-
-    revokeUrl()
-    urlRef.current = url
-    audio.src = url
-    audio.play().then(() => {
-      playingRef.current = true
-    }).catch(() => {
-      playingRef.current = false
+  const playWav = useCallback(
+    (blob: Blob) => {
+      const audio = audioRef.current
+      if (!audio) return
       revokeUrl()
-      playNext()
-    })
-  }, [revokeUrl])
+      const url = URL.createObjectURL(blob)
+      urlRef.current = url
+      audio.src = url
+      audio.play().catch(() => {})
+    },
+    [revokeUrl],
+  )
 
+  // Set up the <audio> element
   useEffect(() => {
     const audio = new Audio()
     audioRef.current = audio
     audio.style.display = 'none'
     document.body.appendChild(audio)
 
-    audio.onended = () => {
-      playingRef.current = false
-      playNext()
-    }
-    audio.onerror = () => {
-      playingRef.current = false
-      playNext()
-    }
-
     return () => {
       audio.pause()
       revokeUrl()
@@ -95,21 +87,25 @@ export function useCompanionAudio() {
         audio.parentNode.removeChild(audio)
       }
       audioRef.current = null
-      queueRef.current = []
-      playingRef.current = false
+      chunksRef.current = []
+      chunksLenRef.current = 0
     }
   }, [revokeUrl])
 
-  const enqueueAudio = useCallback(
-    (pcmData: ArrayBuffer) => {
-      const wav = pcmToWav(pcmData)
-      queueRef.current.push(wav)
-      if (!playingRef.current) {
-        playNext()
-      }
-    },
-    [playNext],
-  )
+  const enqueueAudio = useCallback((pcmData: ArrayBuffer) => {
+    const chunk = new Int16Array(pcmData)
+    if (chunk.length === 0) return
+    chunksRef.current.push(chunk)
+    chunksLenRef.current += chunk.length
+  }, [])
+
+  const flush = useCallback(() => {
+    if (chunksLenRef.current === 0) return
+    const wav = pcmChunksToWav(chunksRef.current)
+    chunksRef.current = []
+    chunksLenRef.current = 0
+    playWav(wav)
+  }, [playWav])
 
   const stopPlayback = useCallback(() => {
     const audio = audioRef.current
@@ -118,27 +114,21 @@ export function useCompanionAudio() {
       revokeUrl()
       audio.src = ''
     }
-    queueRef.current = []
-    playingRef.current = false
+    chunksRef.current = []
+    chunksLenRef.current = 0
   }, [revokeUrl])
 
   const resume = useCallback(() => {
-    const audio = audioRef.current
-    if (!audio) return
-
-    // Unlock audio with a silent WAV
+    // Unlock audio by playing a minimal silent WAV
     const silent = new Int16Array(1)
     silent[0] = 0
-    const wav = pcmToWav(silent.buffer)
-    const url = URL.createObjectURL(wav)
-    revokeUrl()
-    urlRef.current = url
-    audio.src = url
-    audio.play().catch(() => {})
-  }, [revokeUrl])
+    const wav = pcmChunksToWav([silent])
+    playWav(wav)
+  }, [playWav])
 
   return {
     enqueueAudio,
+    flush,
     stopPlayback,
     resume,
   }

desktop/src/hooks/useCompanionWebSocket.ts

@@ -3,6 +3,7 @@ import { useCompanionStore } from '../stores/companionStore'
 
 interface Callbacks {
   onAudioReceived: (data: ArrayBuffer) => void
+  onDone?: () => void
 }
 
 export function useCompanionWebSocket(callbacks: Callbacks) {
@@ -102,6 +103,7 @@ export function useCompanionWebSocket(callbacks: Callbacks) {
               )
               setGenerating(false)
               setSpeaking(false)
+              callbacksRef.current.onDone?.()
               break
           }
         } catch {

desktop/src/lib/audioEngine.ts

@@ -0,0 +1,319 @@
+export type EngineState =
+  | 'uninitialized'
+  | 'initializing'
+  | 'running'
+  | 'suspended'
+  | 'failed'
+  | 'closed'
+  | 'fallback'
+
+// PCM Int16 -> Float32 conversion
+function pcmToFloat32(int16: Int16Array): Float32Array {
+  const float32 = new Float32Array(int16.length)
+  for (let i = 0; i < int16.length; i++) {
+    const sample = int16[i]!
+    float32[i] = sample < 0 ? sample / 0x8000 : sample / 0x7fff
+  }
+  return float32
+}
+
+// PCM Int16 -> WAV Blob (for fallback mode)
+function pcmToWav(pcmBuffer: ArrayBuffer): Blob {
+  const int16 = new Int16Array(pcmBuffer)
+  const numSamples = int16.length
+  if (numSamples === 0) {
+    const empty = new ArrayBuffer(44)
+    const v = new DataView(empty)
+    const w = (o: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)) }
+    w(0, 'RIFF'); v.setUint32(4, 36, true); w(8, 'WAVE')
+    w(12, 'fmt '); v.setUint32(16, 16, true); v.setUint16(20, 1, true)
+    v.setUint16(22, 1, true); v.setUint32(24, 24000, true); v.setUint32(28, 48000, true)
+    v.setUint16(32, 2, true); v.setUint16(34, 16, true)
+    w(36, 'data'); v.setUint32(40, 0, true)
+    return new Blob([empty], { type: 'audio/wav' })
+  }
+
+  const sampleRate = 24000
+  const numChannels = 1
+  const bitsPerSample = 16
+  const blockAlign = (numChannels * bitsPerSample) / 8
+  const byteRate = sampleRate * blockAlign
+  const dataSize = numSamples * blockAlign
+  const totalSize = 44 + dataSize
+
+  const buf = new ArrayBuffer(totalSize)
+  const v = new DataView(buf)
+
+  const w = (o: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)) }
+
+  w(0, 'RIFF')
+  v.setUint32(4, totalSize - 8, true)
+  w(8, 'WAVE')
+  w(12, 'fmt ')
+  v.setUint32(16, 16, true) // PCM
+  v.setUint16(20, 1, true)
+  v.setUint16(22, numChannels, true)
+  v.setUint32(24, sampleRate, true)
+  v.setUint32(28, byteRate, true)
+  v.setUint16(32, blockAlign, true)
+  v.setUint16(34, bitsPerSample, true)
+  w(36, 'data')
+  v.setUint32(40, dataSize, true)
+
+  new Int16Array(buf, 44, numSamples).set(int16)
+  return new Blob([buf], { type: 'audio/wav' })
+}
+
+// Safety limit: 60 seconds = 1,440,000 samples @ 24kHz
+const MAX_FALLBACK_SAMPLES = 24000 * 60
+
+export class AudioEngine {
+  private ctx: AudioContext | null = null
+  private state: EngineState = 'uninitialized'
+  private nextTime: number = 0
+  private keepaliveOsc: OscillatorNode | null = null
+  private keepaliveGain: GainNode | null = null
+
+  // MediaStream keepalive (secondary — most reliable for WebKit)
+  private silentAudioEl: HTMLAudioElement | null = null
+  private mediaStreamDest: MediaStreamAudioDestinationNode | null = null
+
+  // Fallback accumulators
+  private chunks: Int16Array[] = []
+  private totalSamples: number = 0
+
+  // State change callback
+  private onStateChange: ((s: EngineState) => void) | null = null
+
+  constructor(onStateChange?: (s: EngineState) => void) {
+    this.onStateChange = onStateChange || null
+  }
+
+  getState(): EngineState {
+    return this.state
+  }
+
+  private setState(s: EngineState) {
+    this.state = s
+    this.onStateChange?.(s)
+  }
+
+  async initialize(): Promise<boolean> {
+    if (this.state === 'running') return true
+    if (this.state === 'fallback') return false
+
+    this.setState('initializing')
+
+    try {
+      // Create AudioContext (MUST be called during user gesture)
+      const ctx = new (window.AudioContext || (window as any).webkitAudioContext)()
+      this.ctx = ctx
+
+      // Primary keepalive: low-freq oscillator (inaudible, keeps context running)
+      this.keepaliveGain = ctx.createGain()
+      this.keepaliveGain.gain.value = 0.001
+      this.keepaliveOsc = ctx.createOscillator()
+      this.keepaliveOsc.frequency.value = 1
+      this.keepaliveOsc.connect(this.keepaliveGain)
+      this.keepaliveGain.connect(ctx.destination)
+      this.keepaliveOsc.start()
+
+      // Secondary keepalive: MediaStream destination trick
+      // WebKit treats a live MediaStream as non-idle and won't suspend the context
+      try {
+        this.mediaStreamDest = ctx.createMediaStreamDestination()
+        const keepaliveGain2 = ctx.createGain()
+        keepaliveGain2.gain.value = 0
+        keepaliveGain2.connect(this.mediaStreamDest)
+        this.silentAudioEl = new Audio()
+        this.silentAudioEl.srcObject = this.mediaStreamDest.stream
+        this.silentAudioEl.play().catch(() => {})
+      } catch {
+        // MediaStream destination not supported; oscillator keepalive is sufficient
+      }
+
+      this.nextTime = ctx.currentTime
+
+      // Handle state changes
+      ctx.onstatechange = () => this.handleStateChange()
+
+      this.setState('running')
+
+      console.log('[AudioEngine] AudioContext created, keepalive running')
+      return true
+    } catch (e) {
+      console.warn('[AudioEngine] Failed to create AudioContext, switching to fallback mode', e)
+      this.enterFallbackMode()
+      return false
+    }
+  }
+
+  private handleStateChange() {
+    if (!this.ctx) return
+    const ctx = this.ctx
+
+    if (ctx.state === 'suspended') {
+      console.log('[AudioEngine] context suspended, attempting resume')
+      this.setState('suspended')
+      ctx.resume()
+        .then(() => {
+          if (this.ctx?.state === 'running') {
+            this.setState('running')
+            // Reset nextTime after suspension
+            this.nextTime = Math.max(this.nextTime, this.ctx.currentTime)
+          }
+        })
+        .catch(() => {
+          console.warn('[AudioEngine] resume() failed, switching to fallback')
+          this.enterFallbackMode()
+        })
+    } else if (ctx.state === 'closed') {
+      console.warn('[AudioEngine] context closed, switching to fallback')
+      this.enterFallbackMode()
+    } else if (ctx.state === 'running') {
+      this.setState('running')
+    }
+  }
+
+  private enterFallbackMode() {
+    if (this.state === 'fallback') return
+    console.log('[AudioEngine] entering fallback mode')
+    this.setState('fallback')
+    if (this.ctx) {
+      this.ctx.close().catch(() => {})
+      this.ctx = null
+    }
+    if (this.keepaliveOsc) {
+      try { this.keepaliveOsc.stop() } catch {}
+      this.keepaliveOsc = null
+    }
+    if (this.keepaliveGain) {
+      this.keepaliveGain = null
+    }
+    this.cleanupMediaStream()
+  }
+
+  private cleanupMediaStream() {
+    if (this.silentAudioEl) {
+      this.silentAudioEl.pause()
+      this.silentAudioEl.srcObject = null
+      this.silentAudioEl = null
+    }
+    if (this.mediaStreamDest) {
+      this.mediaStreamDest.disconnect()
+      this.mediaStreamDest = null
+    }
+  }
+
+  enqueuePCM(pcmData: ArrayBuffer): void {
+    if (this.state === 'fallback') {
+      this.enqueueFallback(pcmData)
+      return
+    }
+    if (this.state !== 'running' || !this.ctx) {
+      // Drop audio until initialized; user needs to click Connect
+      return
+    }
+
+    const ctx = this.ctx
+    const int16 = new Int16Array(pcmData)
+    const float32 = pcmToFloat32(int16)
+
+    // Apply micro-fades at chunk boundaries to prevent clicking artifacts.
+    // 128 samples @ 24kHz = ~5.3ms fade — inaudible as a fade but eliminates
+    // the DC discontinuity between consecutive scheduled chunks.
+    const FADE_LEN = Math.min(128, float32.length >> 1)
+    if (FADE_LEN > 0) {
+      for (let i = 0; i < FADE_LEN; i++) {
+        const t = i / FADE_LEN
+        float32[i] *= t                         // fade in
+        float32[float32.length - 1 - i] *= t    // fade out
+      }
+    }
+
+    // Create audio buffer
+    const audioBuffer = ctx.createBuffer(1, float32.length, 24000)
+    audioBuffer.getChannelData(0).set(float32)
+
+    // Create source node
+    const source = ctx.createBufferSource()
+    source.buffer = audioBuffer
+
+    // Connect through a gain node for consistency
+    // We'll use the keepalive gain's destination, but create separate gain per buffer
+    // Actually just connect to destination directly; we have keepalive running already
+    source.connect(ctx.destination)
+
+    // Schedule at the correct time for gapless playback
+    const startTime = Math.max(ctx.currentTime, this.nextTime)
+    source.start(startTime)
+
+    // Advance the scheduled time
+    this.nextTime = startTime + audioBuffer.duration
+
+    // Cleanup when playback ends
+    source.onended = () => {
+      try { source.disconnect() } catch {}
+    }
+  }
+
+  private enqueueFallback(pcmData: ArrayBuffer) {
+    const chunk = new Int16Array(pcmData)
+    this.chunks.push(chunk)
+    this.totalSamples += chunk.length
+
+    // Safety: if we exceed 60s, force-flush so user hears SOMETHING
+    if (this.totalSamples >= MAX_FALLBACK_SAMPLES) {
+      console.warn('[AudioEngine] Fallback buffer exceeded 60s, force-flushing')
+      // We don't flush here; flush() is called externally on `done`
+      // The caller needs to decide how to play the WAV
+    }
+  }
+
+  /**
+   * In fallback mode: concatenate all accumulated PCM and return a WAV Blob.
+   * In Web Audio mode: returns null (audio already scheduled).
+   */
+  flush(): Blob | null {
+    if (this.state === 'fallback' && this.totalSamples > 0) {
+      const combined = new Int16Array(this.totalSamples)
+      let offset = 0
+      for (const chunk of this.chunks) {
+        combined.set(chunk, offset)
+        offset += chunk.length
+      }
+      this.chunks = []
+      this.totalSamples = 0
+      return pcmToWav(combined.buffer)
+    }
+    // In Web Audio mode, nothing to do
+    return null
+  }
+
+  stop(): void {
+    // Stop keepalive
+    if (this.keepaliveOsc) {
+      try { this.keepaliveOsc.stop() } catch {}
+      try { this.keepaliveOsc.disconnect() } catch {}
+      this.keepaliveOsc = null
+    }
+    if (this.keepaliveGain) {
+      try { this.keepaliveGain.disconnect() } catch {}
+      this.keepaliveGain = null
+    }
+    this.cleanupMediaStream()
+
+    // Close AudioContext
+    if (this.ctx) {
+      this.ctx.close().catch(() => {})
+      this.ctx = null
+    }
+
+    // Clear fallback accumulators
+    this.chunks = []
+    this.totalSamples = 0
+
+    this.nextTime = 0
+    this.setState('uninitialized')
+  }
+}

desktop/src/pages/Companion.tsx

@@ -29,8 +29,11 @@ export function Companion() {
     },
     [companionAudio],
   )
+  const onDone = useCallback(() => {
+    companionAudio.flush()
+  }, [companionAudio])
 
-  const ws = useCompanionWebSocket({ onAudioReceived })
+  const ws = useCompanionWebSocket({ onAudioReceived, onDone })
   const webcam = useWebcam(ws.connected && cameraEnabled)
   useMicrophone(ws.connected && micEnabled, ws.sendAudio)