omninexus-ai / components /LiveView.tsx
aditya9708's picture
Auto-deploy from AI Studio
e4c7604 verified
Raw History Blame Contribute Delete
10.4 kB
import React, { useState, useEffect, useRef } from 'react';
import { GoogleGenAI, LiveServerMessage, Modality, Blob } from '@google/genai';
const LiveView: React.FC = () => {
const [isActive, setIsActive] = useState(false);
const [transcript, setTranscript] = useState<string[]>([]);
const [isConnecting, setIsConnecting] = useState(false);
const [error, setError] = useState<string | null>(null);
const sessionRef = useRef<any>(null);
const inputAudioContextRef = useRef<AudioContext | null>(null);
const outputAudioContextRef = useRef<AudioContext | null>(null);
const nextStartTimeRef = useRef<number>(0);
const sourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
// Helper: Manual Base64 encoding as per guidelines
const encode = (bytes: Uint8Array) => {
let binary = '';
const len = bytes.byteLength;
for (let i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
};
// Helper: Manual Base64 decoding as per guidelines
const decode = (base64: string) => {
const binaryString = atob(base64);
const len = binaryString.length;
const bytes = new Uint8Array(len);
for (let i = 0; i < len; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes;
};
// Helper: Audio decoding from raw PCM bytes as per guidelines
const decodeAudioData = async (
data: Uint8Array,
ctx: AudioContext,
sampleRate: number,
numChannels: number,
): Promise<AudioBuffer> => {
const dataInt16 = new Int16Array(data.buffer);
const frameCount = dataInt16.length / numChannels;
const buffer = ctx.createBuffer(numChannels, frameCount, sampleRate);
for (let channel = 0; channel < numChannels; channel++) {
const channelData = buffer.getChannelData(channel);
for (let i = 0; i < frameCount; i++) {
channelData[i] = dataInt16[i * numChannels + channel] / 32768.0;
}
}
return buffer;
};
// Helper: Create raw PCM blob for streaming
const createBlob = (data: Float32Array): Blob => {
const l = data.length;
const int16 = new Int16Array(l);
for (let i = 0; i < l; i++) {
int16[i] = data[i] * 32768;
}
return {
data: encode(new Uint8Array(int16.buffer)),
mimeType: 'audio/pcm;rate=16000',
};
};
const stopSession = () => {
if (sessionRef.current) {
sessionRef.current.close?.();
sessionRef.current = null;
}
if (inputAudioContextRef.current) {
inputAudioContextRef.current.close();
inputAudioContextRef.current = null;
}
if (outputAudioContextRef.current) {
outputAudioContextRef.current.close();
outputAudioContextRef.current = null;
}
sourcesRef.current.forEach(source => source.stop());
sourcesRef.current.clear();
setIsActive(false);
setIsConnecting(false);
};
const startSession = async () => {
if (isConnecting) return;
setIsConnecting(true);
setError(null);
try {
// Guidelines: Always create a new GoogleGenAI instance right before the call
const ai = new GoogleGenAI({ apiKey: process.env.API_KEY });
const inputCtx = new (window.AudioContext || (window as any).webkitAudioContext)({sampleRate: 16000});
const outputCtx = new (window.AudioContext || (window as any).webkitAudioContext)({sampleRate: 24000});
inputAudioContextRef.current = inputCtx;
outputAudioContextRef.current = outputCtx;
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
let currentInputTranscription = '';
let currentOutputTranscription = '';
const sessionPromise = ai.live.connect({
model: 'gemini-2.5-flash-native-audio-preview-12-2025',
callbacks: {
onopen: () => {
setIsActive(true);
setIsConnecting(false);
setTranscript(prev => [...prev, "System: Live connection established."]);
const source = inputCtx.createMediaStreamSource(stream);
const scriptProcessor = inputCtx.createScriptProcessor(4096, 1, 1);
scriptProcessor.onaudioprocess = (e) => {
const inputData = e.inputBuffer.getChannelData(0);
const pcmBlob = createBlob(inputData);
// CRITICAL: Initiate sendRealtimeInput after live.connect call resolves.
sessionPromise.then(session => {
session.sendRealtimeInput({ media: pcmBlob });
});
};
source.connect(scriptProcessor);
scriptProcessor.connect(inputCtx.destination);
},
onmessage: async (message: LiveServerMessage) => {
// Handle Transcriptions
if (message.serverContent?.outputTranscription) {
currentOutputTranscription += message.serverContent.outputTranscription.text;
} else if (message.serverContent?.inputTranscription) {
currentInputTranscription += message.serverContent.inputTranscription.text;
}
if (message.serverContent?.turnComplete) {
const userLine = currentInputTranscription;
const aiLine = currentOutputTranscription;
if (userLine) setTranscript(prev => [...prev, `You: ${userLine}`]);
if (aiLine) setTranscript(prev => [...prev, `Gemini: ${aiLine}`]);
currentInputTranscription = '';
currentOutputTranscription = '';
}
// Handle Audio Output
const base64Audio = message.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data;
if (base64Audio) {
nextStartTimeRef.current = Math.max(nextStartTimeRef.current, outputCtx.currentTime);
const audioBuffer = await decodeAudioData(decode(base64Audio), outputCtx, 24000, 1);
const source = outputCtx.createBufferSource();
source.buffer = audioBuffer;
const outputNode = outputCtx.createGain();
source.connect(outputNode);
outputNode.connect(outputCtx.destination);
source.addEventListener('ended', () => {
sourcesRef.current.delete(source);
});
source.start(nextStartTimeRef.current);
nextStartTimeRef.current += audioBuffer.duration;
sourcesRef.current.add(source);
}
// Handle Interruptions
if (message.serverContent?.interrupted) {
sourcesRef.current.forEach(s => s.stop());
sourcesRef.current.clear();
nextStartTimeRef.current = 0;
}
},
onerror: (e) => {
console.error("Live session error", e);
setError("Connection error. Please check your network and API key.");
stopSession();
},
onclose: () => {
stopSession();
}
},
config: {
responseModalities: [Modality.AUDIO], // Array with single Modality.AUDIO element
speechConfig: {
voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Zephyr' } },
},
inputAudioTranscription: {}, // Enable input transcription
outputAudioTranscription: {}, // Enable output transcription
systemInstruction: 'You are OmniNexus Live, a friendly AI conversationalist.'
}
});
sessionRef.current = await sessionPromise;
} catch (err: any) {
console.error(err);
setError("Failed to access microphone or initialize AI session.");
setIsConnecting(false);
}
};
useEffect(() => {
return () => {
stopSession();
};
}, []);
return (
<div className="max-w-2xl mx-auto h-full flex flex-col items-center justify-center p-4">
<div className="relative mb-12">
{isActive && (
<>
<div className="absolute inset-0 bg-indigo-500/20 rounded-full animate-ping scale-150" />
<div className="absolute inset-0 bg-indigo-500/10 rounded-full animate-pulse scale-125" />
</>
)}
<div className={`w-48 h-48 rounded-full flex items-center justify-center transition-all duration-500 ${isActive ? 'bg-indigo-600 shadow-2xl shadow-indigo-600/50' : 'bg-slate-800'}`}>
<svg className={`w-20 h-20 ${isActive ? 'text-white' : 'text-slate-600'}`} fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z" /></svg>
</div>
</div>
<div className="text-center mb-10">
<h2 className="text-3xl font-bold mb-2">{isActive ? 'Gemini is Listening' : 'Real-time Voice'}</h2>
<p className="text-slate-400">Natural, low-latency conversations powered by Gemini Live API.</p>
{error && <p className="text-red-400 mt-2 text-sm font-medium">{error}</p>}
</div>
<div className="w-full bg-slate-900 border border-slate-800 rounded-3xl p-6 h-64 overflow-y-auto mb-8 scroll-smooth flex flex-col">
{transcript.length === 0 ? (
<p className="text-slate-600 italic text-center my-auto">Start a session to begin the conversation...</p>
) : (
transcript.map((line, i) => (
<div key={i} className="mb-2 text-sm leading-relaxed text-slate-300">
<span className="text-indigo-400 font-bold mr-2">●</span>
{line}
</div>
))
)}
</div>
<button
onClick={isActive ? stopSession : startSession}
disabled={isConnecting}
className={`px-12 py-4 rounded-2xl font-bold text-lg transition-all ${isActive ? 'bg-red-500/10 text-red-500 border border-red-500/50 hover:bg-red-500/20' : 'premium-gradient text-white shadow-xl shadow-indigo-600/20'}`}
>
{isConnecting ? 'Initializing...' : isActive ? 'End Voice Session' : 'Start Live Conversation'}
</button>
<p className="mt-6 text-xs text-slate-500 max-w-xs text-center">
Make sure you have selected a valid paid API key for the best performance.
</p>
</div>
);
};
export default LiveView;