import React, { useState, useEffect, useRef } from 'react'; import { GoogleGenAI, LiveServerMessage, Modality, Blob } from '@google/genai'; const LiveView: React.FC = () => { const [isActive, setIsActive] = useState(false); const [transcript, setTranscript] = useState([]); const [isConnecting, setIsConnecting] = useState(false); const [error, setError] = useState(null); const sessionRef = useRef(null); const inputAudioContextRef = useRef(null); const outputAudioContextRef = useRef(null); const nextStartTimeRef = useRef(0); const sourcesRef = useRef>(new Set()); // Helper: Manual Base64 encoding as per guidelines const encode = (bytes: Uint8Array) => { let binary = ''; const len = bytes.byteLength; for (let i = 0; i < len; i++) { binary += String.fromCharCode(bytes[i]); } return btoa(binary); }; // Helper: Manual Base64 decoding as per guidelines const decode = (base64: string) => { const binaryString = atob(base64); const len = binaryString.length; const bytes = new Uint8Array(len); for (let i = 0; i < len; i++) { bytes[i] = binaryString.charCodeAt(i); } return bytes; }; // Helper: Audio decoding from raw PCM bytes as per guidelines const decodeAudioData = async ( data: Uint8Array, ctx: AudioContext, sampleRate: number, numChannels: number, ): Promise => { const dataInt16 = new Int16Array(data.buffer); const frameCount = dataInt16.length / numChannels; const buffer = ctx.createBuffer(numChannels, frameCount, sampleRate); for (let channel = 0; channel < numChannels; channel++) { const channelData = buffer.getChannelData(channel); for (let i = 0; i < frameCount; i++) { channelData[i] = dataInt16[i * numChannels + channel] / 32768.0; } } return buffer; }; // Helper: Create raw PCM blob for streaming const createBlob = (data: Float32Array): Blob => { const l = data.length; const int16 = new Int16Array(l); for (let i = 0; i < l; i++) { int16[i] = data[i] * 32768; } return { data: encode(new Uint8Array(int16.buffer)), mimeType: 'audio/pcm;rate=16000', }; }; const stopSession = () => { if (sessionRef.current) { sessionRef.current.close?.(); sessionRef.current = null; } if (inputAudioContextRef.current) { inputAudioContextRef.current.close(); inputAudioContextRef.current = null; } if (outputAudioContextRef.current) { outputAudioContextRef.current.close(); outputAudioContextRef.current = null; } sourcesRef.current.forEach(source => source.stop()); sourcesRef.current.clear(); setIsActive(false); setIsConnecting(false); }; const startSession = async () => { if (isConnecting) return; setIsConnecting(true); setError(null); try { // Guidelines: Always create a new GoogleGenAI instance right before the call const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); const inputCtx = new (window.AudioContext || (window as any).webkitAudioContext)({sampleRate: 16000}); const outputCtx = new (window.AudioContext || (window as any).webkitAudioContext)({sampleRate: 24000}); inputAudioContextRef.current = inputCtx; outputAudioContextRef.current = outputCtx; const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); let currentInputTranscription = ''; let currentOutputTranscription = ''; const sessionPromise = ai.live.connect({ model: 'gemini-2.5-flash-native-audio-preview-12-2025', callbacks: { onopen: () => { setIsActive(true); setIsConnecting(false); setTranscript(prev => [...prev, "System: Live connection established."]); const source = inputCtx.createMediaStreamSource(stream); const scriptProcessor = inputCtx.createScriptProcessor(4096, 1, 1); scriptProcessor.onaudioprocess = (e) => { const inputData = e.inputBuffer.getChannelData(0); const pcmBlob = createBlob(inputData); // CRITICAL: Initiate sendRealtimeInput after live.connect call resolves. sessionPromise.then(session => { session.sendRealtimeInput({ media: pcmBlob }); }); }; source.connect(scriptProcessor); scriptProcessor.connect(inputCtx.destination); }, onmessage: async (message: LiveServerMessage) => { // Handle Transcriptions if (message.serverContent?.outputTranscription) { currentOutputTranscription += message.serverContent.outputTranscription.text; } else if (message.serverContent?.inputTranscription) { currentInputTranscription += message.serverContent.inputTranscription.text; } if (message.serverContent?.turnComplete) { const userLine = currentInputTranscription; const aiLine = currentOutputTranscription; if (userLine) setTranscript(prev => [...prev, `You: ${userLine}`]); if (aiLine) setTranscript(prev => [...prev, `Gemini: ${aiLine}`]); currentInputTranscription = ''; currentOutputTranscription = ''; } // Handle Audio Output const base64Audio = message.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data; if (base64Audio) { nextStartTimeRef.current = Math.max(nextStartTimeRef.current, outputCtx.currentTime); const audioBuffer = await decodeAudioData(decode(base64Audio), outputCtx, 24000, 1); const source = outputCtx.createBufferSource(); source.buffer = audioBuffer; const outputNode = outputCtx.createGain(); source.connect(outputNode); outputNode.connect(outputCtx.destination); source.addEventListener('ended', () => { sourcesRef.current.delete(source); }); source.start(nextStartTimeRef.current); nextStartTimeRef.current += audioBuffer.duration; sourcesRef.current.add(source); } // Handle Interruptions if (message.serverContent?.interrupted) { sourcesRef.current.forEach(s => s.stop()); sourcesRef.current.clear(); nextStartTimeRef.current = 0; } }, onerror: (e) => { console.error("Live session error", e); setError("Connection error. Please check your network and API key."); stopSession(); }, onclose: () => { stopSession(); } }, config: { responseModalities: [Modality.AUDIO], // Array with single Modality.AUDIO element speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Zephyr' } }, }, inputAudioTranscription: {}, // Enable input transcription outputAudioTranscription: {}, // Enable output transcription systemInstruction: 'You are OmniNexus Live, a friendly AI conversationalist.' } }); sessionRef.current = await sessionPromise; } catch (err: any) { console.error(err); setError("Failed to access microphone or initialize AI session."); setIsConnecting(false); } }; useEffect(() => { return () => { stopSession(); }; }, []); return (
{isActive && ( <>
)}

{isActive ? 'Gemini is Listening' : 'Real-time Voice'}

Natural, low-latency conversations powered by Gemini Live API.

{error &&

{error}

}
{transcript.length === 0 ? (

Start a session to begin the conversation...

) : ( transcript.map((line, i) => (
● {line}
)) )}

Make sure you have selected a valid paid API key for the best performance.

); }; export default LiveView;