Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /MarkinHaus_ToolBoxV2 /llm-gateway /live_handler.py
| """ | |
| Live Voice API Handler - Real-time voice conversation via WebSocket | |
| Features: | |
| - Bidirectional audio streaming (WebSocket) | |
| - Wake word detection (pre/post/mid modes) | |
| - Interrupt handling with [INTERRUPTED BY USER] marker | |
| - Parallel LLM + TTS streaming for minimal latency | |
| - Session management with SQLite persistence | |
| - Uses Ollama backend instead of direct port connections | |
| """ | |
| import asyncio | |
| import base64 | |
| import json | |
| import secrets | |
| import time | |
| import io | |
| import re | |
| from datetime import datetime, timedelta | |
| from typing import Optional, Dict, List, Any, AsyncGenerator | |
| from dataclasses import dataclass, field, asdict | |
| from enum import Enum | |
| import aiosqlite | |
| import httpx | |
| from fastapi import WebSocket, WebSocketDisconnect, HTTPException | |
| from pydantic import BaseModel, Field | |
| # === Enums & Models === | |
| class WakeWordMode(str, Enum): | |
| PRE = "pre" # Wake word at start, discard everything before | |
| POST = "post" # Everything from speech start to wake word goes to LLM | |
| MID = "mid" # Wake word must appear somewhere in utterance | |
| class AudioFormat(str, Enum): | |
| WEBM = "webm" | |
| OPUS = "opus" | |
| WAV = "wav" | |
| PCM = "pcm" | |
| MP3 = "mp3" | |
| class AudioConfig(BaseModel): | |
| input_format: AudioFormat = AudioFormat.WEBM | |
| output_format: AudioFormat = AudioFormat.OPUS | |
| sample_rate: int = 24000 | |
| allow_interrupt: bool = True | |
| class WakeWordConfig(BaseModel): | |
| enabled: bool = False | |
| words: List[str] = Field(default_factory=list) | |
| mode: WakeWordMode = WakeWordMode.PRE | |
| class VoiceConfig(BaseModel): | |
| voice_id: str = "default" | |
| speed: float = 1.0 | |
| language: str = "auto" | |
| class LLMConfig(BaseModel): | |
| model: str | |
| system_prompt: str = "Du bist ein hilfreicher Assistent." | |
| tools: List[Dict] = Field(default_factory=list) | |
| history_length: int = 20 | |
| temperature: float = 0.7 | |
| max_tokens: int = 1024 | |
| class LiveSessionRequest(BaseModel): | |
| audio_config: AudioConfig = Field(default_factory=AudioConfig) | |
| wake_word_config: WakeWordConfig = Field(default_factory=WakeWordConfig) | |
| voice_config: VoiceConfig = Field(default_factory=VoiceConfig) | |
| llm_config: LLMConfig | |
| session_ttl_minutes: int = Field(default=15, ge=5, le=20) | |
| class LiveSessionResponse(BaseModel): | |
| session_token: str | |
| websocket_url: str | |
| expires_at: str | |
| # === Session State === | |
| class ConversationTurn: | |
| role: str # "user" or "assistant" | |
| content: str | |
| timestamp: float = field(default_factory=time.time) | |
| interrupted: bool = False | |
| class LiveSession: | |
| token: str | |
| user_id: int | |
| audio_config: AudioConfig | |
| wake_word_config: WakeWordConfig | |
| voice_config: VoiceConfig | |
| llm_config: LLMConfig | |
| history: List[ConversationTurn] = field(default_factory=list) | |
| created_at: float = field(default_factory=time.time) | |
| last_activity: float = field(default_factory=time.time) | |
| expires_at: float = 0.0 | |
| is_active: bool = True | |
| # Runtime state (not persisted) | |
| is_generating: bool = False | |
| current_output: str = "" | |
| def to_dict(self) -> Dict: | |
| return { | |
| "token": self.token, | |
| "user_id": self.user_id, | |
| "audio_config": self.audio_config.model_dump(), | |
| "wake_word_config": self.wake_word_config.model_dump(), | |
| "voice_config": self.voice_config.model_dump(), | |
| "llm_config": self.llm_config.model_dump(), | |
| "history": [asdict(h) for h in self.history], | |
| "created_at": self.created_at, | |
| "last_activity": self.last_activity, | |
| "expires_at": self.expires_at, | |
| "is_active": self.is_active | |
| } | |
| def from_dict(cls, data: Dict) -> "LiveSession": | |
| return cls( | |
| token=data["token"], | |
| user_id=data["user_id"], | |
| audio_config=AudioConfig(**data["audio_config"]), | |
| wake_word_config=WakeWordConfig(**data["wake_word_config"]), | |
| voice_config=VoiceConfig(**data["voice_config"]), | |
| llm_config=LLMConfig(**data["llm_config"]), | |
| history=[ConversationTurn(**h) for h in data.get("history", [])], | |
| created_at=data["created_at"], | |
| last_activity=data["last_activity"], | |
| expires_at=data["expires_at"], | |
| is_active=data.get("is_active", True) | |
| ) | |
| def get_messages_for_llm(self) -> List[Dict]: | |
| """Build message list for LLM, respecting history_length""" | |
| messages = [] | |
| # System prompt | |
| if self.llm_config.system_prompt: | |
| messages.append({ | |
| "role": "system", | |
| "content": self.llm_config.system_prompt | |
| }) | |
| # Conversation history (last N turns) | |
| history_slice = self.history[-self.llm_config.history_length:] | |
| for turn in history_slice: | |
| content = turn.content | |
| if turn.interrupted: | |
| content += " [INTERRUPTED BY USER]" | |
| messages.append({ | |
| "role": turn.role, | |
| "content": content | |
| }) | |
| return messages | |
| def add_user_turn(self, text: str): | |
| """Add user message to history""" | |
| self.history.append(ConversationTurn(role="user", content=text)) | |
| self.last_activity = time.time() | |
| def add_assistant_turn(self, text: str, interrupted: bool = False): | |
| """Add assistant message to history""" | |
| self.history.append(ConversationTurn( | |
| role="assistant", | |
| content=text, | |
| interrupted=interrupted | |
| )) | |
| self.last_activity = time.time() | |
| # === Live Handler === | |
| class LiveHandler: | |
| """ | |
| Manages live voice sessions with WebSocket connections. | |
| Architecture: | |
| - Sessions stored in SQLite for persistence | |
| - In-memory cache for active sessions | |
| - Lazy cleanup on new session creation | |
| - Uses Ollama backend for LLM and TTS | |
| """ | |
| def __init__(self, db_path: str, model_manager): | |
| self.db_path = db_path | |
| self.model_manager = model_manager | |
| self.active_sessions: Dict[str, LiveSession] = {} | |
| self.active_websockets: Dict[str, WebSocket] = {} | |
| async def init_db(self): | |
| """Initialize live_sessions table""" | |
| async with aiosqlite.connect(self.db_path) as db: | |
| await db.execute(""" | |
| CREATE TABLE IF NOT EXISTS live_sessions | |
| ( | |
| token | |
| TEXT | |
| PRIMARY | |
| KEY, | |
| user_id | |
| INTEGER | |
| NOT | |
| NULL, | |
| session_data | |
| TEXT | |
| NOT | |
| NULL, | |
| created_at | |
| TEXT | |
| DEFAULT | |
| CURRENT_TIMESTAMP, | |
| expires_at | |
| TEXT | |
| NOT | |
| NULL, | |
| is_active | |
| INTEGER | |
| DEFAULT | |
| 1, | |
| FOREIGN | |
| KEY | |
| ( | |
| user_id | |
| ) REFERENCES users | |
| ( | |
| id | |
| ) | |
| ) | |
| """) | |
| await db.execute(""" | |
| CREATE INDEX IF NOT EXISTS idx_live_sessions_user | |
| ON live_sessions(user_id) | |
| """) | |
| await db.execute(""" | |
| CREATE INDEX IF NOT EXISTS idx_live_sessions_expires | |
| ON live_sessions(expires_at) | |
| """) | |
| await db.commit() | |
| async def _cleanup_expired_sessions(self, user_id: int): | |
| """Remove expired sessions for user (lazy cleanup)""" | |
| now = datetime.utcnow().isoformat() | |
| async with aiosqlite.connect(self.db_path) as db: | |
| # Get expired session tokens | |
| cursor = await db.execute( | |
| "SELECT token FROM live_sessions WHERE user_id = ? AND expires_at < ?", | |
| (user_id, now) | |
| ) | |
| expired = [row[0] for row in await cursor.fetchall()] | |
| # Remove from memory | |
| for token in expired: | |
| self.active_sessions.pop(token, None) | |
| ws = self.active_websockets.pop(token, None) | |
| if ws: | |
| try: | |
| await ws.close(code=1000, reason="Session expired") | |
| except: | |
| pass | |
| # Remove from DB | |
| await db.execute( | |
| "DELETE FROM live_sessions WHERE user_id = ? AND expires_at < ?", | |
| (user_id, now) | |
| ) | |
| await db.commit() | |
| async def create_session( | |
| self, | |
| request: LiveSessionRequest, | |
| user_id: int | |
| ) -> LiveSessionResponse: | |
| """Create new live session""" | |
| # Verify TTS model is available | |
| try: | |
| tts_model = self.model_manager.find_tts_model() | |
| except Exception: | |
| tts_model = None | |
| if not tts_model: | |
| raise HTTPException( | |
| 503, | |
| "No TTS model loaded. Load a TTS model (type='tts') first." | |
| ) | |
| # Verify LLM model is available | |
| try: | |
| llm_model = self.model_manager.find_model(request.llm_config.model) | |
| except Exception: | |
| try: | |
| llm_model = self.model_manager.find_text_model() | |
| except Exception: | |
| llm_model = None | |
| if not llm_model: | |
| raise HTTPException( | |
| 503, | |
| f"No LLM model available. Load model '{request.llm_config.model}' or any text model." | |
| ) | |
| # Cleanup expired sessions for this user | |
| await self._cleanup_expired_sessions(user_id) | |
| # Generate session token | |
| token = f"live-{secrets.token_hex(16)}" | |
| # Calculate expiry | |
| ttl_minutes = max(5, min(20, request.session_ttl_minutes)) | |
| expires_at = time.time() + (ttl_minutes * 60) | |
| # Create session | |
| session = LiveSession( | |
| token=token, | |
| user_id=user_id, | |
| audio_config=request.audio_config, | |
| wake_word_config=request.wake_word_config, | |
| voice_config=request.voice_config, | |
| llm_config=request.llm_config, | |
| expires_at=expires_at | |
| ) | |
| # Store in memory and DB | |
| self.active_sessions[token] = session | |
| async with aiosqlite.connect(self.db_path) as db: | |
| await db.execute( | |
| """INSERT INTO live_sessions | |
| (token, user_id, session_data, expires_at) | |
| VALUES (?, ?, ?, ?)""", | |
| ( | |
| token, | |
| user_id, | |
| json.dumps(session.to_dict()), | |
| datetime.fromtimestamp(expires_at).isoformat() | |
| ) | |
| ) | |
| await db.commit() | |
| return LiveSessionResponse( | |
| session_token=token, | |
| websocket_url=f"/v1/audio/live/ws/{token}", | |
| expires_at=datetime.fromtimestamp(expires_at).isoformat() | |
| ) | |
| async def get_session(self, token: str) -> Optional[LiveSession]: | |
| """Get session by token""" | |
| # Check memory first | |
| if token in self.active_sessions: | |
| session = self.active_sessions[token] | |
| if time.time() < session.expires_at: | |
| return session | |
| else: | |
| # Expired, remove | |
| del self.active_sessions[token] | |
| return None | |
| # Load from DB | |
| async with aiosqlite.connect(self.db_path) as db: | |
| db.row_factory = aiosqlite.Row | |
| cursor = await db.execute( | |
| "SELECT * FROM live_sessions WHERE token = ? AND is_active = 1", | |
| (token,) | |
| ) | |
| row = await cursor.fetchone() | |
| if not row: | |
| return None | |
| # Check expiry | |
| expires_at = datetime.fromisoformat(row["expires_at"]) | |
| if datetime.utcnow() > expires_at: | |
| return None | |
| # Reconstruct session | |
| session_data = json.loads(row["session_data"]) | |
| session = LiveSession.from_dict(session_data) | |
| # Cache in memory | |
| self.active_sessions[token] = session | |
| return session | |
| async def update_session(self, session: LiveSession): | |
| """Persist session changes to DB""" | |
| async with aiosqlite.connect(self.db_path) as db: | |
| await db.execute( | |
| "UPDATE live_sessions SET session_data = ? WHERE token = ?", | |
| (json.dumps(session.to_dict()), session.token) | |
| ) | |
| await db.commit() | |
| async def close_session(self, token: str): | |
| """Close and cleanup session""" | |
| self.active_sessions.pop(token, None) | |
| ws = self.active_websockets.pop(token, None) | |
| if ws: | |
| try: | |
| await ws.close(code=1000, reason="Session closed") | |
| except: | |
| pass | |
| async with aiosqlite.connect(self.db_path) as db: | |
| await db.execute( | |
| "UPDATE live_sessions SET is_active = 0 WHERE token = ?", | |
| (token,) | |
| ) | |
| await db.commit() | |
| # === WebSocket Handler === | |
| async def handle_websocket(self, websocket: WebSocket, token: str): | |
| """Main WebSocket handler for live session""" | |
| # Validate session | |
| session = await self.get_session(token) | |
| if not session: | |
| await websocket.close(code=4004, reason="Invalid or expired session") | |
| return | |
| # Accept connection | |
| await websocket.accept() | |
| self.active_websockets[token] = websocket | |
| # Send ready message | |
| await websocket.send_json({ | |
| "type": "ready", | |
| "session_token": token, | |
| "config": { | |
| "audio": session.audio_config.model_dump(), | |
| "wake_word": session.wake_word_config.model_dump(), | |
| "voice": session.voice_config.model_dump() | |
| } | |
| }) | |
| try: | |
| while True: | |
| # Receive message | |
| message = await websocket.receive_json() | |
| msg_type = message.get("type") | |
| if msg_type == "audio": | |
| # Client sends audio chunk with transcription | |
| await self._handle_audio_message(websocket, session, message) | |
| elif msg_type == "end_turn": | |
| # Client signals end of speech | |
| await self._handle_end_turn(websocket, session, message) | |
| elif msg_type == "interrupt": | |
| # Client interrupts current output | |
| await self._handle_interrupt(websocket, session) | |
| elif msg_type == "ping": | |
| # Keep-alive | |
| session.last_activity = time.time() | |
| await websocket.send_json({"type": "pong"}) | |
| elif msg_type == "close": | |
| # Client requests close | |
| break | |
| except WebSocketDisconnect: | |
| pass | |
| except Exception as e: | |
| try: | |
| await websocket.send_json({ | |
| "type": "error", | |
| "message": str(e) | |
| }) | |
| except: | |
| pass | |
| finally: | |
| self.active_websockets.pop(token, None) | |
| await self.update_session(session) | |
| async def _handle_audio_message( | |
| self, | |
| websocket: WebSocket, | |
| session: LiveSession, | |
| message: Dict | |
| ): | |
| """Handle incoming audio chunk with transcription from client""" | |
| # Client does VAD, so we receive transcribed text | |
| # Audio is optional (for potential server-side processing) | |
| transcript = message.get("transcript", "") | |
| is_final = message.get("is_final", False) | |
| if transcript: | |
| # Send acknowledgment | |
| await websocket.send_json({ | |
| "type": "transcript", | |
| "text": transcript, | |
| "is_input": True, | |
| "is_final": is_final | |
| }) | |
| async def _handle_end_turn( | |
| self, | |
| websocket: WebSocket, | |
| session: LiveSession, | |
| message: Dict | |
| ): | |
| """Handle end of user turn - process with LLM and TTS""" | |
| transcript = message.get("transcript", "") | |
| if not transcript.strip(): | |
| await websocket.send_json({ | |
| "type": "error", | |
| "message": "Empty transcript" | |
| }) | |
| return | |
| # Process wake word if enabled | |
| processed_text = self._process_wake_word(transcript, session.wake_word_config) | |
| if processed_text is None: | |
| # Wake word not found when required | |
| await websocket.send_json({ | |
| "type": "wake_word_required", | |
| "message": "Wake word not detected" | |
| }) | |
| return | |
| if not processed_text.strip(): | |
| await websocket.send_json({ | |
| "type": "error", | |
| "message": "No content after wake word processing" | |
| }) | |
| return | |
| # Add to history | |
| session.add_user_turn(processed_text) | |
| session.is_generating = True | |
| session.current_output = "" | |
| # Track usage | |
| start_time = time.time() | |
| tokens_in = 0 | |
| tokens_out = 0 | |
| tools_called = [] | |
| try: | |
| # Get Ollama URL | |
| ollama_url = self.model_manager.ollama_url | |
| # Build LLM request | |
| messages = session.get_messages_for_llm() | |
| llm_url = f"{ollama_url}/v1/chat/completions" | |
| tts_url = f"{ollama_url}/v1/audio/speech" | |
| # Stream LLM response | |
| full_response = "" | |
| sentence_buffer = "" | |
| interrupted = False | |
| async with httpx.AsyncClient(timeout=120.0) as client: | |
| payload = { | |
| "model": session.llm_config.model, | |
| "messages": messages, | |
| "temperature": session.llm_config.temperature, | |
| "max_tokens": session.llm_config.max_tokens, | |
| "stream": True | |
| } | |
| if session.llm_config.tools: | |
| payload["tools"] = session.llm_config.tools | |
| async with client.stream("POST", llm_url, json=payload) as resp: | |
| async for line in resp.aiter_lines(): | |
| # Check for interrupt | |
| if not session.is_generating: | |
| interrupted = True | |
| break | |
| if not line.startswith("data: "): | |
| continue | |
| data = line[6:] | |
| if data == "[DONE]": | |
| break | |
| try: | |
| chunk = json.loads(data) | |
| # Handle tool calls | |
| if chunk.get("choices", [{}])[0].get("delta", {}).get("tool_calls"): | |
| tool_call = chunk["choices"][0]["delta"]["tool_calls"][0] | |
| if tool_call.get("function", {}).get("name"): | |
| tools_called.append(tool_call["function"]["name"]) | |
| # Get content | |
| content = chunk.get("choices", [{}])[0].get("delta", {}).get("content", "") | |
| if content: | |
| full_response += content | |
| session.current_output = full_response | |
| sentence_buffer += content | |
| tokens_out += 1 | |
| # Send text chunk to client | |
| await websocket.send_json({ | |
| "type": "text", | |
| "text": content, | |
| "is_final": False | |
| }) | |
| # Check for complete sentence for TTS | |
| sentences = self._extract_sentences(sentence_buffer) | |
| for sentence in sentences[:-1]: # All complete sentences | |
| if sentence.strip(): | |
| # Generate and stream TTS | |
| await self._stream_tts( | |
| websocket, | |
| tts_url, | |
| sentence.strip(), | |
| session | |
| ) | |
| # Keep incomplete sentence in buffer | |
| sentence_buffer = sentences[-1] if sentences else "" | |
| # Usage info | |
| if chunk.get("usage"): | |
| tokens_in = chunk["usage"].get("prompt_tokens", tokens_in) | |
| tokens_out = chunk["usage"].get("completion_tokens", tokens_out) | |
| except json.JSONDecodeError: | |
| continue | |
| # Process remaining buffer | |
| if sentence_buffer.strip() and session.is_generating: | |
| await self._stream_tts( | |
| websocket, | |
| tts_url, | |
| sentence_buffer.strip(), | |
| session | |
| ) | |
| # Add to history | |
| session.add_assistant_turn(full_response, interrupted=interrupted) | |
| # Calculate cost | |
| pricing = {"input_per_1k": 0.0001, "output_per_1k": 0.0002} | |
| cost = (tokens_in / 1000 * pricing["input_per_1k"] + | |
| tokens_out / 1000 * pricing["output_per_1k"]) | |
| # Send turn complete | |
| await websocket.send_json({ | |
| "type": "turn_complete", | |
| "input_transcript": processed_text, | |
| "output_transcript": full_response, | |
| "tools_called": tools_called, | |
| "interrupted": interrupted, | |
| "usage": { | |
| "tokens_in": tokens_in, | |
| "tokens_out": tokens_out | |
| }, | |
| "cost": cost, | |
| "latency_ms": int((time.time() - start_time) * 1000) | |
| }) | |
| except Exception as e: | |
| await websocket.send_json({ | |
| "type": "error", | |
| "message": str(e) | |
| }) | |
| finally: | |
| session.is_generating = False | |
| session.current_output = "" | |
| await self.update_session(session) | |
| async def _handle_interrupt(self, websocket: WebSocket, session: LiveSession): | |
| """Handle interrupt from client""" | |
| if not session.is_generating: | |
| return | |
| # Stop generation | |
| session.is_generating = False | |
| partial_output = session.current_output | |
| # Send interrupted acknowledgment | |
| await websocket.send_json({ | |
| "type": "interrupted", | |
| "partial_output": partial_output | |
| }) | |
| def _process_wake_word( | |
| self, | |
| text: str, | |
| config: WakeWordConfig | |
| ) -> Optional[str]: | |
| """ | |
| Process wake word based on mode. | |
| Returns processed text or None if wake word required but not found. | |
| """ | |
| if not config.enabled or not config.words: | |
| return text | |
| text_lower = text.lower() | |
| found_word = None | |
| found_pos = -1 | |
| # Find wake word | |
| for word in config.words: | |
| pos = text_lower.find(word.lower()) | |
| if pos != -1: | |
| found_word = word | |
| found_pos = pos | |
| break | |
| if found_word is None: | |
| return None # Wake word required but not found | |
| word_end = found_pos + len(found_word) | |
| if config.mode == WakeWordMode.PRE: | |
| # Wake word must be at start (with some tolerance) | |
| # Return everything after the wake word | |
| if found_pos <= 10: # Allow up to 10 chars before (filler words) | |
| return text[word_end:].strip() | |
| return None | |
| elif config.mode == WakeWordMode.POST: | |
| # Return everything from start to wake word | |
| return text[:found_pos].strip() | |
| elif config.mode == WakeWordMode.MID: | |
| # Wake word just needs to be present | |
| # Return full text without the wake word | |
| result = text[:found_pos] + text[word_end:] | |
| # Clean up double spaces | |
| return ' '.join(result.split()) | |
| return text | |
| def _extract_sentences(self, text: str) -> List[str]: | |
| """Split text into sentences for TTS streaming""" | |
| # Split on sentence boundaries | |
| pattern = r'(?<=[.!?])\s+' | |
| parts = re.split(pattern, text) | |
| return parts if parts else [text] | |
| async def _stream_tts( | |
| self, | |
| websocket: WebSocket, | |
| tts_url: str, | |
| text: str, | |
| session: LiveSession | |
| ): | |
| """Generate TTS and stream audio to client using Ollama""" | |
| if not session.is_generating: | |
| return | |
| try: | |
| async with httpx.AsyncClient(timeout=60.0) as client: | |
| # TTS request (OpenAI compatible) | |
| payload = { | |
| "model": "tts", | |
| "input": text, | |
| "voice": session.voice_config.voice_id, | |
| "speed": session.voice_config.speed, | |
| "response_format": session.audio_config.output_format.value | |
| } | |
| # Stream TTS response | |
| async with client.stream("POST", tts_url, json=payload) as resp: | |
| if resp.status_code != 200: | |
| return | |
| async for chunk in resp.aiter_bytes(chunk_size=4096): | |
| if not session.is_generating: | |
| break | |
| # Send audio chunk | |
| await websocket.send_json({ | |
| "type": "audio", | |
| "audio": base64.b64encode(chunk).decode(), | |
| "format": session.audio_config.output_format.value, | |
| "text": text | |
| }) | |
| except Exception as e: | |
| # TTS error - continue without audio | |
| print(f"TTS error: {e}") | |
| # === Factory Function === | |
| def create_live_handler(db_path: str, model_manager) -> LiveHandler: | |
| """Create and return LiveHandler instance""" | |
| return LiveHandler(db_path, model_manager) | |
Xet Storage Details
- Size:
- 27.7 kB
- Xet hash:
- f030361cb3abc51b5398ea58be3b091736701452d69d83505406ba6a6f9315cb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.