Spaces:
Runtime error
Runtime error
Download bot.py from Smartwaylabs/pipecat-livekit: direct link, hf CLI and curl.
- Browser
- Download file 20.3 kB
-
https://huggingface.co/spaces/Smartwaylabs/pipecat-livekit/resolve/main/bot.py
- Command line
-
hf download hf://spaces/Smartwaylabs/pipecat-livekit/bot.py
-
curl -L -o bot.py https://huggingface.co/spaces/Smartwaylabs/pipecat-livekit/resolve/main/bot.py
20.3 kB
| # | |
| # Copyright (c) 2025, Daily | |
| # | |
| # SPDX-License-Identifier: BSD 2-Clause License | |
| # | |
| """ | |
| Pipecat Voice AI Bot - Main entry point. | |
| This module orchestrates the voice AI pipeline with: | |
| - Real-time audio processing (STT → LLM → TTS) | |
| - RTVI protocol for client communication | |
| - Flow management for conversation state | |
| - Transcript and analytics collection | |
| Supports multiple transports: | |
| - Daily.co (WebRTC/RTVI) | |
| - Twilio (Media Streams WebSocket) | |
| """ | |
| import asyncio | |
| import os | |
| from typing import Any, Optional | |
| import sentry_sdk | |
| from dotenv import load_dotenv | |
| # Langfuse tracing | |
| from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter | |
| from pipecat.utils.tracing.setup import setup_tracing | |
| from pipecat.pipeline.runner import PipelineRunner | |
| # Local modules | |
| from bot_schemas import BotData | |
| from bot_config import BotConfig, TransportType | |
| from services import create_llm_service, create_tts_service, create_stt_service, create_transport, create_livekit_transport | |
| from utils.postgres import postgres | |
| from core.pipeline import PipelineBuilder | |
| from core.rtvi import setup_rtvi_processor, setup_client_ready_handler | |
| from core.events import setup_event_handlers | |
| from core.flow import end_conversation_handler, process_flow_config_handlers | |
| from core.transcript import TranscriptCollector, TranscriptObserver | |
| from local.recording import create_recording_processors | |
| from utils.logging import log_info, log_warning, log_error | |
| from fastapi import WebSocket | |
| from metrics import ( | |
| capture_event, | |
| get_default_provider, | |
| create_llm_observer, | |
| create_latency_observer, | |
| ) | |
| from pipecat_flows import FlowConfig, FlowManager | |
| from pipecatcloud.agent import DailySessionArguments, WebSocketSessionArguments | |
| load_dotenv(override=True) | |
| def create_transcript_handlers(log_key: str, enabled: bool = True): | |
| """Create transcript handlers for collecting conversation.""" | |
| if not enabled: | |
| log_info(log_key, "Transcript collection disabled") | |
| return None, [] | |
| transcript_collector = TranscriptCollector() | |
| transcript_observer = TranscriptObserver(transcript_collector) | |
| log_info(log_key, "Transcript collection enabled") | |
| return transcript_collector, [transcript_observer] | |
| def setup_tracing_if_enabled(log_key: str) -> None: | |
| """Configure Langfuse tracing if enabled via environment.""" | |
| if os.getenv("ENABLE_TRACING", "false").lower() != "true": | |
| return | |
| try: | |
| otlp_exporter = OTLPSpanExporter() | |
| setup_tracing( | |
| service_name="pipecat-bot", | |
| exporter=otlp_exporter, | |
| console_export=os.getenv("OTEL_CONSOLE_EXPORT", "false").lower() == "true", | |
| ) | |
| log_info(log_key, "Langfuse tracing configured") | |
| except Exception as e: | |
| log_error(log_key, f"Failed to configure tracing: {e}") | |
| def setup_sentry(environment: str) -> None: | |
| """Initialize Sentry error tracking (optional).""" | |
| sentry_dsn = os.environ.get("SENTRY_DSN") | |
| if not sentry_dsn: | |
| return # Sentry is optional | |
| sentry_sdk.init( | |
| dsn=sentry_dsn, | |
| traces_sample_rate=1.0, | |
| environment=environment, | |
| ) | |
| async def setup_flow_manager( | |
| payload: BotData, | |
| rtvi_client: Any, | |
| task: Any, | |
| llm: Any, | |
| context_aggregator: Any, | |
| log_key: str, | |
| ) -> Optional[FlowManager]: | |
| """Set up FlowManager for conversation flow.""" | |
| if not payload.flow: | |
| log_error(log_key, "No flow provided") | |
| return None | |
| flow_config = payload.flow.model_dump() | |
| flow_config_obj = process_flow_config_handlers( | |
| FlowConfig(**flow_config), | |
| rtvi_client, | |
| ) | |
| flow_manager = FlowManager( | |
| task=task, | |
| llm=llm, | |
| context_aggregator=context_aggregator, | |
| flow_config=flow_config_obj, | |
| ) | |
| flow_manager.log_key = log_key | |
| flow_manager.register_action("end_conversation", end_conversation_handler) | |
| # Handle legacy initial_node attribute | |
| if not getattr(flow_manager, "initial_node", None): | |
| legacy_initial = getattr(flow_manager, "_initial_node", None) | |
| if legacy_initial: | |
| setattr(flow_manager, "initial_node", legacy_initial) | |
| return flow_manager | |
| async def enforce_max_session_duration(timeout_seconds: int, task: Any, log_key: str): | |
| """Cancel task after max session duration.""" | |
| try: | |
| await asyncio.sleep(timeout_seconds) | |
| log_warning(log_key, f"Max session duration ({timeout_seconds}s) reached") | |
| await task.cancel() | |
| except asyncio.CancelledError: | |
| pass | |
| async def run_pipeline(config: BotConfig) -> dict: | |
| """ | |
| Unified pipeline for all transport types. | |
| Returns dict with session results (transcript, duration, etc.) | |
| """ | |
| result = { | |
| "transcript_messages": [], | |
| "duration": 0, | |
| "status": "COMPLETED", | |
| } | |
| # Parse bot data | |
| if config.bot_data: | |
| data = BotData(**config.bot_data) if isinstance(config.bot_data, dict) else config.bot_data | |
| elif config.agent_id: | |
| # Load from database | |
| bot_data_dict = await postgres.get_bot_data(config.agent_id) | |
| if not bot_data_dict: | |
| raise ValueError(f"Failed to load bot data for agent: {config.agent_id}") | |
| data = BotData(**bot_data_dict) | |
| else: | |
| raise ValueError("Either bot_data or agent_id must be provided") | |
| # Extract configuration | |
| transcript_key = data.metadata.transcript_key if hasattr(data.metadata, 'transcript_key') else None | |
| environment = data.metadata.environment or "production" | |
| user_id = data.metadata.user_id or config.caller_number or f"user-{config.session_id}" | |
| max_duration = data.metadata.max_duration or 3600 | |
| # Create log key | |
| if config.transport_type == TransportType.TWILIO: | |
| log_key = f"twilio_{config.call_sid or config.session_id or 'unknown'}" | |
| else: | |
| log_key = transcript_key.replace("/", "_") if transcript_key else config.session_id or "unknown-session" | |
| log_info(log_key, f"Pipeline started [transport={config.transport_type.value}, env={environment}]") | |
| # Setup infrastructure | |
| setup_tracing_if_enabled(log_key) | |
| setup_sentry(environment) | |
| # Create or use provided transport | |
| if config.transport_type == TransportType.TWILIO: | |
| if not config.transport: | |
| raise ValueError("Twilio transport must be provided via config.transport") | |
| transport = config.transport | |
| rtvi = None # No RTVI for Twilio | |
| elif config.transport_type == TransportType.LIVEKIT: | |
| # LiveKit Cloud transport | |
| if not config.livekit_url or not config.livekit_token: | |
| raise ValueError("livekit_url and livekit_token required for LiveKit transport") | |
| transport = create_livekit_transport( | |
| room_url=config.livekit_url, | |
| token=config.livekit_token, | |
| room_name=config.livekit_room_name or "", | |
| participant_identity=config.participant_identity, | |
| ) | |
| rtvi = None # RTVI not supported with LiveKit yet | |
| log_info(log_key, f"LiveKit transport created [identity={config.participant_identity}]") | |
| else: | |
| # Daily.co transport | |
| if not config.room_url or not config.token: | |
| raise ValueError("room_url and token required for Daily transport") | |
| transport = create_transport(config.room_url, config.token, config.bot_name) | |
| rtvi = None | |
| if config.rtvi_enabled: | |
| rtvi = setup_rtvi_processor( | |
| pipeline_getter=lambda: pipeline, | |
| log_func=lambda msg: log_info(log_key, msg), | |
| ) | |
| # Create services with provider selection | |
| llm = create_llm_service( | |
| provider=data.config.llm_provider.value, | |
| model=data.config.model, | |
| ) | |
| tts = create_tts_service( | |
| provider=data.config.tts_provider.value, | |
| voice_id=data.config.voice_id, | |
| ) | |
| stt = create_stt_service( | |
| provider=data.config.stt_provider.value, | |
| ) | |
| log_info( | |
| log_key, | |
| f"Services initialized [llm={data.config.llm_provider.value}/{data.config.model}, " | |
| f"tts={data.config.tts_provider.value}, stt={data.config.stt_provider.value}]" | |
| ) | |
| # Setup recording (local only for now) | |
| audio_processor = None | |
| if config.local_recording: | |
| audio_processor = create_recording_processors(config.session_id, config.recording_dir) | |
| log_info(log_key, f"Local recording mode [dir={config.recording_dir}]") | |
| # Setup transcript collection - always enabled for Twilio to capture messages | |
| enable_transcript = config.transport_type == TransportType.TWILIO or transcript_key is not None | |
| transcript_collector, transcript_observers = create_transcript_handlers(log_key, enable_transcript) | |
| # Build pipeline | |
| pipeline = None | |
| builder = ( | |
| PipelineBuilder() | |
| .with_transport(transport) | |
| .with_rtvi(rtvi) | |
| .with_stt(stt) | |
| .with_llm(llm) | |
| .with_tts(tts) | |
| .with_recording(audio_processor) | |
| .with_transcript() | |
| ) | |
| pipeline = builder.build() | |
| # Build observers | |
| llm_observer = None | |
| latency_observer = None | |
| if config.transport_type == TransportType.DAILY: | |
| llm_observer = create_llm_observer(data, llm.model_name) | |
| latency_observer = create_latency_observer(data, rtvi if config.rtvi_enabled else None) | |
| observers = builder.build_observers(transcript_observers, llm_observer, latency_observer) | |
| # Create task | |
| task = builder.create_task(pipeline, observers, log_key, user_id, environment) | |
| # Setup flow manager | |
| flow_manager = await setup_flow_manager( | |
| data, rtvi, task, llm, builder.context_aggregator, log_key | |
| ) | |
| if not flow_manager: | |
| log_error(log_key, "Failed to initialize FlowManager") | |
| result["status"] = "FAILED" | |
| return result | |
| log_info(log_key, "FlowManager initialized") | |
| # Setup max session duration | |
| log_info(log_key, f"Max duration: {max_duration}s") | |
| asyncio.create_task(enforce_max_session_duration(max_duration, task, log_key)) | |
| # Setup event handlers based on transport type | |
| if config.transport_type == TransportType.TWILIO: | |
| # For Twilio, setup simplified event handlers | |
| setup_twilio_event_handlers( | |
| transport=transport, | |
| task=task, | |
| flow_manager=flow_manager, | |
| transcript_processor=builder.transcript_processor, | |
| transcript_collector=transcript_collector, | |
| log_key=log_key, | |
| ) | |
| elif config.transport_type == TransportType.LIVEKIT: | |
| # LiveKit event handlers | |
| setup_livekit_event_handlers( | |
| transport=transport, | |
| task=task, | |
| flow_manager=flow_manager, | |
| transcript_processor=builder.transcript_processor, | |
| transcript_collector=transcript_collector, | |
| log_key=log_key, | |
| ) | |
| else: | |
| # Daily.co event handlers | |
| if config.rtvi_enabled and rtvi: | |
| setup_client_ready_handler(rtvi, lambda msg: log_info(log_key, msg)) | |
| setup_event_handlers( | |
| transport=transport, | |
| task=task, | |
| flow_manager=flow_manager, | |
| rtvi=rtvi, | |
| transcript=builder.transcript_processor, | |
| transcript_collector=transcript_collector, | |
| audio_processor=audio_processor, | |
| config=config, | |
| log_key=log_key, | |
| capture_event_func=capture_event, | |
| bot_data=data, | |
| ) | |
| # Run pipeline | |
| runner = PipelineRunner() | |
| try: | |
| await runner.run(task) | |
| log_info(log_key, "Pipeline completed successfully") | |
| result["status"] = "COMPLETED" | |
| except Exception as e: | |
| log_error(log_key, f"Pipeline failed: {e}") | |
| result["status"] = "FAILED" | |
| if config.transport_type == TransportType.DAILY: | |
| capture_event(data, "pipecat_error_encountered", {"message": str(e)}) | |
| raise | |
| finally: | |
| # Collect transcript messages | |
| if transcript_collector: | |
| result["transcript_messages"] = transcript_collector.get_messages() | |
| if config.transport_type == TransportType.DAILY: | |
| get_default_provider().close() | |
| log_info(log_key, "Pipeline finished") | |
| return result | |
| def setup_twilio_event_handlers( | |
| transport: Any, | |
| task: Any, | |
| flow_manager: Any, | |
| transcript_processor: Any, | |
| transcript_collector: Optional[TranscriptCollector], | |
| log_key: str, | |
| ) -> None: | |
| """ | |
| Setup event handlers for Twilio transport. | |
| Twilio doesn't have participant join/leave events like Daily.co, | |
| so we handle the flow differently. | |
| """ | |
| from datetime import datetime, timezone | |
| # Initialize flow immediately when transport is ready | |
| async def handle_client_connected(transport_instance, websocket): | |
| log_info(log_key, "Twilio client connected") | |
| # Start transcript session | |
| if transcript_collector: | |
| recording_start_time = datetime.now(timezone.utc).isoformat() | |
| transcript_collector.start_session(recording_start_time) | |
| # Initialize flow | |
| await flow_manager.initialize() | |
| # Set initial node | |
| if flow_manager.initial_node: | |
| from core.flow import send_state_sync | |
| await send_state_sync(flow_manager, flow_manager.initial_node) | |
| async def handle_client_disconnected(transport_instance, websocket): | |
| log_info(log_key, "Twilio client disconnected") | |
| # Finalize transcript | |
| if transcript_collector: | |
| transcript_collector.finalize() | |
| # Cancel task | |
| await task.cancel() | |
| # Handle transcript updates | |
| if transcript_processor: | |
| async def handle_transcript_update(processor, frame): | |
| if transcript_collector: | |
| for message in frame.messages: | |
| transcript_collector.process_utterance(message.role, message.content) | |
| def setup_livekit_event_handlers( | |
| transport: Any, | |
| task: Any, | |
| flow_manager: Any, | |
| transcript_processor: Any, | |
| transcript_collector: Optional[TranscriptCollector], | |
| log_key: str, | |
| ) -> None: | |
| """ | |
| Setup event handlers for LiveKit transport. | |
| LiveKit has similar event model to Daily.co but with different event names. | |
| """ | |
| from datetime import datetime, timezone | |
| async def handle_participant_connected(transport_instance, participant): | |
| log_info(log_key, f"LiveKit participant connected: {participant.identity if hasattr(participant, 'identity') else participant}") | |
| # Start transcript session | |
| if transcript_collector: | |
| recording_start_time = datetime.now(timezone.utc).isoformat() | |
| transcript_collector.start_session(recording_start_time) | |
| # Initialize flow | |
| await flow_manager.initialize() | |
| # Set initial node | |
| if flow_manager.initial_node: | |
| from core.flow import send_state_sync | |
| await send_state_sync(flow_manager, flow_manager.initial_node) | |
| async def handle_participant_disconnected(transport_instance, participant): | |
| log_info(log_key, f"LiveKit participant disconnected: {participant.identity if hasattr(participant, 'identity') else participant}") | |
| # Finalize transcript | |
| if transcript_collector: | |
| transcript_collector.finalize() | |
| # Cancel task | |
| await task.cancel() | |
| # Handle transcript updates if processor is available | |
| if transcript_processor: | |
| async def handle_transcript_update(processor, frame): | |
| if transcript_collector: | |
| for message in frame.messages: | |
| transcript_collector.process_utterance(message.role, message.content) | |
| async def run_livekit_pipeline(config: BotConfig) -> dict: | |
| """ | |
| Entry point for LiveKit bot sessions. | |
| This is called by the LiveKit runner to start a bot with LiveKit transport. | |
| """ | |
| log_info(config.session_id or "livekit", f"Starting LiveKit pipeline [url={config.livekit_url}]") | |
| return await run_pipeline(config) | |
| # ----------------------------------------------------------------------------- | |
| # Entry Points | |
| # ----------------------------------------------------------------------------- | |
| async def bot(args): | |
| """ | |
| Main bot entry point for Pipecat Cloud. | |
| Handles both transport types: | |
| - DailySessionArguments: WebRTC sessions via Daily.co | |
| - WebSocketSessionArguments: Twilio Media Streams via WebSocket | |
| """ | |
| # Check argument type to determine transport | |
| if isinstance(args, WebSocketSessionArguments): | |
| # Twilio WebSocket transport | |
| log_info("", f"Twilio WebSocket session started [session_id={args.session_id}]") | |
| try: | |
| from services.twilio_transport import handle_twilio_call | |
| # Pass session_id to enable Pipecat Cloud logs retrieval | |
| await handle_twilio_call(args.websocket, run_twilio_pipeline_wrapper, args.session_id) | |
| log_info("", "Twilio WebSocket session completed") | |
| except Exception as e: | |
| log_error("", f"Twilio WebSocket session error: {str(e)}") | |
| raise | |
| elif isinstance(args, DailySessionArguments): | |
| # Daily.co WebRTC transport | |
| log_info("", f"Bot initialized [room={args.room_url}]") | |
| if not args.body: | |
| raise ValueError("No data provided") | |
| try: | |
| session_id = args.session_id if hasattr(args, "session_id") else None | |
| config = BotConfig( | |
| transport_type=TransportType.DAILY, | |
| session_id=session_id, | |
| room_url=args.room_url, | |
| token=args.token, | |
| bot_data=args.body, | |
| ) | |
| await run_pipeline(config) | |
| log_info(None, "Bot completed") | |
| except Exception as e: | |
| log_error(None, f"Bot error: {str(e)}") | |
| raise | |
| else: | |
| raise TypeError(f"Unsupported session arguments type: {type(args)}") | |
| async def run_twilio_pipeline_wrapper( | |
| transport: Any, | |
| agent_config: dict, | |
| context: Any, # TwilioCallContext | |
| ) -> None: | |
| """ | |
| Wrapper to run unified pipeline for Twilio calls. | |
| This bridges the old TwilioCallContext API with the new unified pipeline. | |
| """ | |
| log_key = f"twilio_{context.call_sid}" | |
| # Load botData from agent config | |
| bot_data_json = agent_config.get("botData") | |
| if bot_data_json: | |
| log_info(log_key, "Using botData from database") | |
| bot_data = bot_data_json | |
| else: | |
| # Fallback to legacy generation | |
| log_warning(log_key, "No botData in database, using legacy generation") | |
| bot_data = postgres._generate_bot_data_legacy(agent_config) | |
| # Create unified config | |
| config = BotConfig( | |
| transport_type=TransportType.TWILIO, | |
| session_id=context.pipecat_session_id, | |
| call_sid=context.call_sid, | |
| stream_sid=context.stream_sid, | |
| caller_number=context.caller_number, | |
| agent_id=context.agent_id, | |
| bot_data=bot_data, | |
| transport=transport, | |
| rtvi_enabled=False, # No RTVI for Twilio | |
| ) | |
| # Run unified pipeline | |
| result = await run_pipeline(config) | |
| # Update context with results for database save | |
| if result.get("transcript_messages"): | |
| context.transcript_messages = result["transcript_messages"] | |
| async def twilio_websocket_handler(websocket: WebSocket) -> None: | |
| """ | |
| FastAPI WebSocket endpoint handler for Twilio Media Streams. | |
| This is for local development. Mount this in your FastAPI app: | |
| @app.websocket("/twilio") | |
| async def twilio_ws(websocket: WebSocket): | |
| await twilio_websocket_handler(websocket) | |
| """ | |
| from services.twilio_transport import handle_twilio_call | |
| await handle_twilio_call(websocket, run_twilio_pipeline_wrapper) | |
| # Legacy alias for backwards compatibility | |
| main = run_pipeline | |