Spaces:
Sleeping
Sleeping
Download components/LiveView.tsx from aditya9708/omninexus-ai: direct link, hf CLI and curl.
- Browser
- Download file 10.4 kB
-
https://huggingface.co/spaces/aditya9708/omninexus-ai/resolve/main/components/LiveView.tsx
- Command line
-
hf download hf://spaces/aditya9708/omninexus-ai/components/LiveView.tsx
-
curl -L -o LiveView.tsx https://huggingface.co/spaces/aditya9708/omninexus-ai/resolve/main/components/LiveView.tsx
10.4 kB
| import React, { useState, useEffect, useRef } from 'react'; | |
| import { GoogleGenAI, LiveServerMessage, Modality, Blob } from '@google/genai'; | |
| const LiveView: React.FC = () => { | |
| const [isActive, setIsActive] = useState(false); | |
| const [transcript, setTranscript] = useState<string[]>([]); | |
| const [isConnecting, setIsConnecting] = useState(false); | |
| const [error, setError] = useState<string | null>(null); | |
| const sessionRef = useRef<any>(null); | |
| const inputAudioContextRef = useRef<AudioContext | null>(null); | |
| const outputAudioContextRef = useRef<AudioContext | null>(null); | |
| const nextStartTimeRef = useRef<number>(0); | |
| const sourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set()); | |
| // Helper: Manual Base64 encoding as per guidelines | |
| const encode = (bytes: Uint8Array) => { | |
| let binary = ''; | |
| const len = bytes.byteLength; | |
| for (let i = 0; i < len; i++) { | |
| binary += String.fromCharCode(bytes[i]); | |
| } | |
| return btoa(binary); | |
| }; | |
| // Helper: Manual Base64 decoding as per guidelines | |
| const decode = (base64: string) => { | |
| const binaryString = atob(base64); | |
| const len = binaryString.length; | |
| const bytes = new Uint8Array(len); | |
| for (let i = 0; i < len; i++) { | |
| bytes[i] = binaryString.charCodeAt(i); | |
| } | |
| return bytes; | |
| }; | |
| // Helper: Audio decoding from raw PCM bytes as per guidelines | |
| const decodeAudioData = async ( | |
| data: Uint8Array, | |
| ctx: AudioContext, | |
| sampleRate: number, | |
| numChannels: number, | |
| ): Promise<AudioBuffer> => { | |
| const dataInt16 = new Int16Array(data.buffer); | |
| const frameCount = dataInt16.length / numChannels; | |
| const buffer = ctx.createBuffer(numChannels, frameCount, sampleRate); | |
| for (let channel = 0; channel < numChannels; channel++) { | |
| const channelData = buffer.getChannelData(channel); | |
| for (let i = 0; i < frameCount; i++) { | |
| channelData[i] = dataInt16[i * numChannels + channel] / 32768.0; | |
| } | |
| } | |
| return buffer; | |
| }; | |
| // Helper: Create raw PCM blob for streaming | |
| const createBlob = (data: Float32Array): Blob => { | |
| const l = data.length; | |
| const int16 = new Int16Array(l); | |
| for (let i = 0; i < l; i++) { | |
| int16[i] = data[i] * 32768; | |
| } | |
| return { | |
| data: encode(new Uint8Array(int16.buffer)), | |
| mimeType: 'audio/pcm;rate=16000', | |
| }; | |
| }; | |
| const stopSession = () => { | |
| if (sessionRef.current) { | |
| sessionRef.current.close?.(); | |
| sessionRef.current = null; | |
| } | |
| if (inputAudioContextRef.current) { | |
| inputAudioContextRef.current.close(); | |
| inputAudioContextRef.current = null; | |
| } | |
| if (outputAudioContextRef.current) { | |
| outputAudioContextRef.current.close(); | |
| outputAudioContextRef.current = null; | |
| } | |
| sourcesRef.current.forEach(source => source.stop()); | |
| sourcesRef.current.clear(); | |
| setIsActive(false); | |
| setIsConnecting(false); | |
| }; | |
| const startSession = async () => { | |
| if (isConnecting) return; | |
| setIsConnecting(true); | |
| setError(null); | |
| try { | |
| // Guidelines: Always create a new GoogleGenAI instance right before the call | |
| const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); | |
| const inputCtx = new (window.AudioContext || (window as any).webkitAudioContext)({sampleRate: 16000}); | |
| const outputCtx = new (window.AudioContext || (window as any).webkitAudioContext)({sampleRate: 24000}); | |
| inputAudioContextRef.current = inputCtx; | |
| outputAudioContextRef.current = outputCtx; | |
| const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); | |
| let currentInputTranscription = ''; | |
| let currentOutputTranscription = ''; | |
| const sessionPromise = ai.live.connect({ | |
| model: 'gemini-2.5-flash-native-audio-preview-12-2025', | |
| callbacks: { | |
| onopen: () => { | |
| setIsActive(true); | |
| setIsConnecting(false); | |
| setTranscript(prev => [...prev, "System: Live connection established."]); | |
| const source = inputCtx.createMediaStreamSource(stream); | |
| const scriptProcessor = inputCtx.createScriptProcessor(4096, 1, 1); | |
| scriptProcessor.onaudioprocess = (e) => { | |
| const inputData = e.inputBuffer.getChannelData(0); | |
| const pcmBlob = createBlob(inputData); | |
| // CRITICAL: Initiate sendRealtimeInput after live.connect call resolves. | |
| sessionPromise.then(session => { | |
| session.sendRealtimeInput({ media: pcmBlob }); | |
| }); | |
| }; | |
| source.connect(scriptProcessor); | |
| scriptProcessor.connect(inputCtx.destination); | |
| }, | |
| onmessage: async (message: LiveServerMessage) => { | |
| // Handle Transcriptions | |
| if (message.serverContent?.outputTranscription) { | |
| currentOutputTranscription += message.serverContent.outputTranscription.text; | |
| } else if (message.serverContent?.inputTranscription) { | |
| currentInputTranscription += message.serverContent.inputTranscription.text; | |
| } | |
| if (message.serverContent?.turnComplete) { | |
| const userLine = currentInputTranscription; | |
| const aiLine = currentOutputTranscription; | |
| if (userLine) setTranscript(prev => [...prev, `You: ${userLine}`]); | |
| if (aiLine) setTranscript(prev => [...prev, `Gemini: ${aiLine}`]); | |
| currentInputTranscription = ''; | |
| currentOutputTranscription = ''; | |
| } | |
| // Handle Audio Output | |
| const base64Audio = message.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data; | |
| if (base64Audio) { | |
| nextStartTimeRef.current = Math.max(nextStartTimeRef.current, outputCtx.currentTime); | |
| const audioBuffer = await decodeAudioData(decode(base64Audio), outputCtx, 24000, 1); | |
| const source = outputCtx.createBufferSource(); | |
| source.buffer = audioBuffer; | |
| const outputNode = outputCtx.createGain(); | |
| source.connect(outputNode); | |
| outputNode.connect(outputCtx.destination); | |
| source.addEventListener('ended', () => { | |
| sourcesRef.current.delete(source); | |
| }); | |
| source.start(nextStartTimeRef.current); | |
| nextStartTimeRef.current += audioBuffer.duration; | |
| sourcesRef.current.add(source); | |
| } | |
| // Handle Interruptions | |
| if (message.serverContent?.interrupted) { | |
| sourcesRef.current.forEach(s => s.stop()); | |
| sourcesRef.current.clear(); | |
| nextStartTimeRef.current = 0; | |
| } | |
| }, | |
| onerror: (e) => { | |
| console.error("Live session error", e); | |
| setError("Connection error. Please check your network and API key."); | |
| stopSession(); | |
| }, | |
| onclose: () => { | |
| stopSession(); | |
| } | |
| }, | |
| config: { | |
| responseModalities: [Modality.AUDIO], // Array with single Modality.AUDIO element | |
| speechConfig: { | |
| voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Zephyr' } }, | |
| }, | |
| inputAudioTranscription: {}, // Enable input transcription | |
| outputAudioTranscription: {}, // Enable output transcription | |
| systemInstruction: 'You are OmniNexus Live, a friendly AI conversationalist.' | |
| } | |
| }); | |
| sessionRef.current = await sessionPromise; | |
| } catch (err: any) { | |
| console.error(err); | |
| setError("Failed to access microphone or initialize AI session."); | |
| setIsConnecting(false); | |
| } | |
| }; | |
| useEffect(() => { | |
| return () => { | |
| stopSession(); | |
| }; | |
| }, []); | |
| return ( | |
| <div className="max-w-2xl mx-auto h-full flex flex-col items-center justify-center p-4"> | |
| <div className="relative mb-12"> | |
| {isActive && ( | |
| <> | |
| <div className="absolute inset-0 bg-indigo-500/20 rounded-full animate-ping scale-150" /> | |
| <div className="absolute inset-0 bg-indigo-500/10 rounded-full animate-pulse scale-125" /> | |
| </> | |
| )} | |
| <div className={`w-48 h-48 rounded-full flex items-center justify-center transition-all duration-500 ${isActive ? 'bg-indigo-600 shadow-2xl shadow-indigo-600/50' : 'bg-slate-800'}`}> | |
| <svg className={`w-20 h-20 ${isActive ? 'text-white' : 'text-slate-600'}`} fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z" /></svg> | |
| </div> | |
| </div> | |
| <div className="text-center mb-10"> | |
| <h2 className="text-3xl font-bold mb-2">{isActive ? 'Gemini is Listening' : 'Real-time Voice'}</h2> | |
| <p className="text-slate-400">Natural, low-latency conversations powered by Gemini Live API.</p> | |
| {error && <p className="text-red-400 mt-2 text-sm font-medium">{error}</p>} | |
| </div> | |
| <div className="w-full bg-slate-900 border border-slate-800 rounded-3xl p-6 h-64 overflow-y-auto mb-8 scroll-smooth flex flex-col"> | |
| {transcript.length === 0 ? ( | |
| <p className="text-slate-600 italic text-center my-auto">Start a session to begin the conversation...</p> | |
| ) : ( | |
| transcript.map((line, i) => ( | |
| <div key={i} className="mb-2 text-sm leading-relaxed text-slate-300"> | |
| <span className="text-indigo-400 font-bold mr-2">●</span> | |
| {line} | |
| </div> | |
| )) | |
| )} | |
| </div> | |
| <button | |
| onClick={isActive ? stopSession : startSession} | |
| disabled={isConnecting} | |
| className={`px-12 py-4 rounded-2xl font-bold text-lg transition-all ${isActive ? 'bg-red-500/10 text-red-500 border border-red-500/50 hover:bg-red-500/20' : 'premium-gradient text-white shadow-xl shadow-indigo-600/20'}`} | |
| > | |
| {isConnecting ? 'Initializing...' : isActive ? 'End Voice Session' : 'Start Live Conversation'} | |
| </button> | |
| <p className="mt-6 text-xs text-slate-500 max-w-xs text-center"> | |
| Make sure you have selected a valid paid API key for the best performance. | |
| </p> | |
| </div> | |
| ); | |
| }; | |
| export default LiveView; | |