Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /CIRISAI_CIRISAgent /tests /test_llm_bus.py
"""
Comprehensive tests for LLM Bus architecture
Tests:
- Multi-provider scenarios
- Provider failover mechanisms
- Load distribution strategies (round-robin, latency-based, random, least-loaded)
- Circuit breaker functionality
- Health checks
- Service registration/deregistration
- Priority-based selection
- Thread safety and concurrent access patterns
"""
import asyncio
import random
from datetime import datetime, timedelta, timezone
from typing import List, Tuple, Type
from unittest.mock import patch
import pytest
from pydantic import BaseModel
from ciris_engine.logic.buses.llm_bus import DistributionStrategy, LLMBus
from ciris_engine.logic.registries.base import Priority, ServiceRegistry
from ciris_engine.logic.registries.circuit_breaker import CircuitState
from ciris_engine.schemas.runtime.enums import ServiceType
from ciris_engine.schemas.runtime.resources import ResourceUsage
class MockResponseModel(BaseModel):
"""Mock response model for testing (not a test class - renamed to avoid pytest collection)"""
message: str
confidence: float = 1.0
class MockTimeService:
"""Mock time service for testing"""
def __init__(self):
self.current_time = datetime.now(timezone.utc)
self.frozen = False
def now(self) -> datetime:
if self.frozen:
return self.current_time
return datetime.now(timezone.utc)
def timestamp(self) -> float:
return self.now().timestamp()
def freeze(self, dt: datetime = None):
self.frozen = True
if dt:
self.current_time = dt
def advance(self, seconds: float):
if self.frozen:
self.current_time += timedelta(seconds=seconds)
class MockLLMService:
"""Mock LLM service for testing"""
def __init__(self, name: str, latency_ms: float = 100, failure_rate: float = 0.0):
self.name = name
self.latency_ms = latency_ms
self.failure_rate = failure_rate
self.call_count = 0
self.healthy = True
self.capabilities = ["call_llm_structured", "get_available_models"]
async def call_llm_structured(
self,
messages: List[dict],
response_model: Type[BaseModel],
max_tokens: int = 1024,
temperature: float = 0.0,
**kwargs,
) -> Tuple[BaseModel, ResourceUsage]:
"""Simulate LLM call with configurable latency and failure"""
self.call_count += 1
# Simulate latency
await asyncio.sleep(self.latency_ms / 1000)
# Simulate failures
if random.random() < self.failure_rate:
raise RuntimeError(f"{self.name} simulated failure")
# Return test response
response = MockResponseModel(message=f"Response from {self.name}")
usage = ResourceUsage(
tokens_used=100,
tokens_input=50,
tokens_output=50,
cost_cents=0.01,
model_used=f"{self.name}-model",
carbon_grams=0.001,
energy_kwh=0.0001,
)
return response, usage
async def is_healthy(self) -> bool:
return self.healthy
async def get_available_models(self) -> List[str]:
return [f"{self.name}-model-1", f"{self.name}-model-2"]
def get_capabilities(self):
"""Return capabilities object"""
class Capabilities:
supports_operation_list = ["call_llm_structured", "get_available_models"]
return Capabilities()
class MockTelemetryService:
"""Mock telemetry service for testing"""
def __init__(self):
self.metrics = []
async def record_metric(self, metric_name: str, value: float, handler_name: str, tags: dict = None):
self.metrics.append({"name": metric_name, "value": value, "handler": handler_name, "tags": tags or {}})
@pytest.fixture
def time_service():
"""Provide mock time service"""
return MockTimeService()
@pytest.fixture
def telemetry_service():
"""Provide mock telemetry service"""
return MockTelemetryService()
@pytest.fixture(scope="function")
def service_registry():
"""Provide service registry with isolated scope per test"""
return ServiceRegistry()
@pytest.fixture
def llm_bus(service_registry, time_service, telemetry_service):
"""Provide LLM bus instance"""
bus = LLMBus(
service_registry=service_registry,
time_service=time_service,
telemetry_service=telemetry_service,
distribution_strategy=DistributionStrategy.ROUND_ROBIN,
circuit_breaker_config={"failure_threshold": 3, "recovery_timeout": 5.0, "half_open_max_calls": 2},
)
# Clear any existing state
bus.clear_circuit_breakers()
return bus
class TestLLMBusBasics:
"""Test basic LLM bus functionality"""
@pytest.mark.asyncio
async def test_single_provider_success(self, llm_bus, service_registry):
"""Test successful call with single provider"""
# Register a mock LLM service
mock_service = MockLLMService("TestLLM", latency_ms=50)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=mock_service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"}, # Mark as mock provider
)
# Make a call
messages = [{"role": "user", "content": "Test"}]
result, usage = await llm_bus.call_llm_structured(
messages=messages, response_model=MockResponseModel, handler_name="test"
)
assert isinstance(result, MockResponseModel)
assert result.message == "Response from TestLLM"
assert usage.tokens_used == 100
assert mock_service.call_count == 1
@pytest.mark.asyncio
async def test_no_providers_error(self, llm_bus):
"""Test error when no providers available"""
with pytest.raises(RuntimeError, match="No LLM services available"):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel, handler_name="test"
)
@pytest.mark.asyncio
async def test_circuit_breaker_skips_to_lower_priority(self, llm_bus, service_registry):
"""Test that services with open circuit breakers are skipped in favor of lower priority services"""
# Register healthy and unhealthy services
healthy_service = MockLLMService("HealthyLLM")
failing_service = MockLLMService("FailingLLM", failure_rate=1.0)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=failing_service,
priority=Priority.HIGH, # Higher priority but will fail
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=healthy_service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Make a call - should try high priority, fail, then use normal priority
result, _ = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel
)
assert result.message == "Response from HealthyLLM"
assert failing_service.call_count == 1 # High priority tried once per retry config (1 retry)
assert healthy_service.call_count == 1 # Normal priority succeeded
class TestDistributionStrategies:
"""Test different load distribution strategies"""
@pytest.mark.asyncio
async def test_round_robin_distribution(self, service_registry, time_service, telemetry_service):
"""Test round-robin distribution across providers"""
# Create bus with round-robin strategy
bus = LLMBus(
service_registry=service_registry,
time_service=time_service,
telemetry_service=telemetry_service,
distribution_strategy=DistributionStrategy.ROUND_ROBIN,
)
# Register multiple services at same priority
services = [MockLLMService(f"LLM-{i}") for i in range(3)]
for service in services:
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Make multiple calls
for i in range(6):
result, _ = await bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i}"}], response_model=MockResponseModel
)
# Should cycle through services
expected_service = services[i % 3]
assert result.message == f"Response from {expected_service.name}"
# Check call counts
for service in services:
assert service.call_count == 2
@pytest.mark.asyncio
async def test_latency_based_distribution(self, service_registry, time_service, telemetry_service):
"""Test latency-based distribution"""
# Create bus with latency-based strategy
bus = LLMBus(
service_registry=service_registry,
time_service=time_service,
telemetry_service=telemetry_service,
distribution_strategy=DistributionStrategy.LATENCY_BASED,
)
# Register services with different latencies
fast_service = MockLLMService("FastLLM", latency_ms=10)
slow_service = MockLLMService("SlowLLM", latency_ms=200)
for service in [fast_service, slow_service]:
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Make initial calls to build up latency metrics
for i in range(2):
await bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test warmup {i}"}], response_model=MockResponseModel
)
# Reset call counts
initial_fast_count = fast_service.call_count
initial_slow_count = slow_service.call_count
# Subsequent calls should prefer fast service based on latency
fast_service_calls = 0
for i in range(5):
result, _ = await bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i+2}"}], response_model=MockResponseModel
)
if "FastLLM" in result.message:
fast_service_calls += 1
# Fast service should get most of the calls after warmup
assert fast_service_calls >= 4 # At least 4 out of 5 calls
@pytest.mark.asyncio
async def test_least_loaded_distribution(self, service_registry, time_service, telemetry_service):
"""Test least-loaded distribution strategy"""
# Create bus with least-loaded strategy
bus = LLMBus(
service_registry=service_registry,
time_service=time_service,
telemetry_service=telemetry_service,
distribution_strategy=DistributionStrategy.LEAST_LOADED,
)
# Register multiple services
services = [MockLLMService(f"LLM-{i}") for i in range(3)]
for service in services:
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Make calls - should distribute evenly
for i in range(6):
await bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i}"}], response_model=MockResponseModel
)
# All services should have equal call counts
assert all(s.call_count == 2 for s in services)
class TestCircuitBreaker:
"""Test circuit breaker functionality"""
@pytest.mark.asyncio
async def test_circuit_breaker_opens_on_failures(self, llm_bus, service_registry, time_service):
"""Test circuit breaker opens after threshold failures"""
# Register a failing service
failing_service = MockLLMService("FailingLLM", failure_rate=1.0)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=failing_service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Try multiple calls - should fail
for i in range(3):
with pytest.raises(RuntimeError, match="All LLM services failed"):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i}"}], response_model=MockResponseModel
)
# Circuit breaker should be open
cb = llm_bus.circuit_breakers[f"MockLLMService_{id(failing_service)}"]
assert cb.state == CircuitState.OPEN
# Further calls should fail immediately without calling service
initial_count = failing_service.call_count
with pytest.raises(RuntimeError):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel
)
assert failing_service.call_count == initial_count # No new calls
@pytest.mark.asyncio
async def test_circuit_breaker_recovery(self, llm_bus, service_registry, time_service):
"""Test circuit breaker recovery after timeout"""
# Use real time patching since circuit breaker uses time.time()
import time as time_module
# Register service that fails initially then recovers
service = MockLLMService("RecoveringLLM")
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Cause failures to open circuit
service.failure_rate = 1.0
for i in range(3):
with pytest.raises(RuntimeError):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i}"}], response_model=MockResponseModel
)
# Verify circuit is open
cb = llm_bus.circuit_breakers[f"MockLLMService_{id(service)}"]
assert cb.state == CircuitState.OPEN
# Service recovers
service.failure_rate = 0.0
# Mock time to simulate recovery timeout passing
original_time = time_module.time
with patch("time.time", return_value=original_time() + 6.0):
# Circuit should allow half-open attempts
result, _ = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test recovery"}], response_model=MockResponseModel
)
assert result.message == "Response from RecoveringLLM"
# After successful calls, circuit should close
for i in range(2):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i}"}], response_model=MockResponseModel
)
assert cb.state == CircuitState.CLOSED
class TestPriorityFailover:
"""Test priority-based failover mechanisms"""
@pytest.mark.asyncio
async def test_failover_by_priority(self, llm_bus, service_registry):
"""Test failover from high to low priority providers"""
# Register services at different priorities
high_priority = MockLLMService("HighPriority", failure_rate=1.0)
normal_priority = MockLLMService("NormalPriority", failure_rate=0.0)
low_priority = MockLLMService("LowPriority", failure_rate=0.0)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=high_priority,
priority=Priority.HIGH,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=normal_priority,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=low_priority,
priority=Priority.LOW,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Call should try high priority first, fail, then use normal priority
result, _ = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel
)
assert result.message == "Response from NormalPriority"
assert high_priority.call_count == 1 # Tried once per retry config (1 retry)
assert normal_priority.call_count == 1 # Succeeded
assert low_priority.call_count == 0 # Not needed
@pytest.mark.asyncio
async def test_all_priorities_fail(self, llm_bus, service_registry, caplog):
"""Test error when all priority levels fail, and verify log verbosity reduction"""
import logging
caplog.set_level(logging.WARNING) # Capture both WARNING and ERROR
# Register failing services at all priorities
for priority in [Priority.HIGH, Priority.NORMAL, Priority.LOW]:
service = MockLLMService(f"{priority.name}LLM", failure_rate=1.0)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=priority,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# First call - should log full errors for each service
with pytest.raises(RuntimeError, match="All LLM services failed"):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel
)
# Count ERROR level logs from first attempt (one per service after retries)
first_error_count = sum(1 for record in caplog.records if record.levelname == "ERROR")
assert first_error_count == 3, "Should have 3 ERROR logs (one per failing service after retries)"
# Clear log capture
caplog.clear()
# Second call - should log warnings instead of full errors (log verbosity reduction)
with pytest.raises(RuntimeError, match="All LLM services failed"):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel
)
# Second attempt should have 0 ERROR logs (all converted to WARNING due to log dedup)
second_error_count = sum(1 for record in caplog.records if record.levelname == "ERROR")
# Filter out the unrelated "no streaming_step on the call stack"
# warning that fires once per LLM_CALL broadcast outside a wrapped
# handler (this test invokes LLMBus directly, not through
# streaming_step). The bus-level retry warnings are what this test
# is counting.
second_warning_count = sum(
1
for record in caplog.records
if record.levelname == "WARNING" and "no streaming_step on the call stack" not in record.message
)
assert second_error_count == 0, "Should have 0 ERROR logs on repeated failures"
# With 1 retry: 3 services × (1 exhausted warning + 1 repeated warning) = 6
assert second_warning_count == 6, "Should have 6 WARNING logs (2 per service with 1 retry)"
class TestConcurrentAccess:
"""Test thread safety and concurrent access patterns"""
@pytest.mark.asyncio
async def test_concurrent_requests(self, llm_bus, service_registry):
"""Test multiple concurrent requests"""
# Register multiple services
services = [MockLLMService(f"LLM-{i}", latency_ms=50) for i in range(3)]
for service in services:
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Make concurrent requests
tasks = []
for i in range(10):
task = llm_bus.call_llm_structured(
messages=[{"role": "user", "content": f"Concurrent test {i}"}], response_model=MockResponseModel
)
tasks.append(task)
# Wait for all to complete
results = await asyncio.gather(*tasks)
# All should succeed
assert len(results) == 10
for result, _ in results:
assert isinstance(result, MockResponseModel)
# Calls should be distributed
total_calls = sum(s.call_count for s in services)
assert total_calls == 10
@pytest.mark.asyncio
async def test_concurrent_registration(self, llm_bus, service_registry):
"""Test concurrent service registration"""
async def register_service(name: str, delay: float):
await asyncio.sleep(delay)
service = MockLLMService(name)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
return service
# Register services concurrently
tasks = [register_service(f"Concurrent-{i}", i * 0.01) for i in range(5)]
services = await asyncio.gather(*tasks)
# All should be registered
assert len(services) == 5
# Should be able to use any of them
result, _ = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel
)
assert "Concurrent" in result.message
class TestTelemetry:
"""Test telemetry and metrics collection"""
@pytest.mark.asyncio
async def test_telemetry_recording(self, llm_bus, service_registry):
"""Test that telemetry is properly recorded"""
# Register service
service = MockLLMService("TelemetryLLM")
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Make calls
for i in range(3):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i}"}],
response_model=MockResponseModel,
handler_name="test_handler",
)
# Check telemetry
telemetry = llm_bus.telemetry_service
assert len(telemetry.metrics) > 0
# Should have token, cost, latency metrics
metric_names = {m["name"] for m in telemetry.metrics}
assert "llm.tokens.total" in metric_names
assert "llm.tokens.input" in metric_names
assert "llm.tokens.output" in metric_names
assert "llm.cost.cents" in metric_names
assert "llm.latency.ms" in metric_names
# Check tags
for metric in telemetry.metrics:
if metric["name"] == "llm.tokens.total":
assert metric["handler"] == "test_handler"
assert metric["tags"]["service"] == f"MockLLMService_{id(service)}"
assert metric["tags"]["model"] == "TelemetryLLM-model"
@pytest.mark.asyncio
async def test_service_stats(self, llm_bus, service_registry):
"""Test service statistics tracking"""
# Register services
fast_service = MockLLMService("Fast", latency_ms=10)
slow_service = MockLLMService("Slow", latency_ms=100, failure_rate=0.2)
for service in [fast_service, slow_service]:
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Make multiple calls
success_count = 0
failure_count = 0
for i in range(10):
try:
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": f"Test {i}"}], response_model=MockResponseModel
)
success_count += 1
except RuntimeError:
failure_count += 1
# Get stats
stats = llm_bus.get_service_stats()
# Should have stats for both services
assert len(stats) >= 2
# Check stats structure
for service_name, service_stats in stats.items():
assert "total_requests" in service_stats
assert "failed_requests" in service_stats
assert "failure_rate" in service_stats
assert "average_latency_ms" in service_stats
assert "circuit_breaker_state" in service_stats
class TestMessageNormalization:
"""Test message normalization and None value filtering"""
@pytest.mark.asyncio
async def test_none_values_stripped_from_llm_messages(self, llm_bus, service_registry):
"""Test that None values are stripped from LLMMessage objects during normalization"""
from ciris_engine.schemas.services.llm import LLMMessage
# Register a mock service that can inspect the messages it receives
received_messages = []
class InspectingLLMService(MockLLMService):
async def call_llm_structured(
self,
messages: List[dict],
response_model: Type[BaseModel],
max_tokens: int = 1024,
temperature: float = 0.0,
**kwargs,
) -> Tuple[BaseModel, ResourceUsage]:
# Capture the messages for inspection
received_messages.extend(messages)
return await super().call_llm_structured(messages, response_model, max_tokens, temperature, **kwargs)
service = InspectingLLMService("Inspector")
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Create LLMMessage with name=None (default)
messages = [
LLMMessage(role="system", content="You are a helpful assistant"), # name will be None by default
LLMMessage(role="user", content="Hello"),
]
# Make the call
await llm_bus.call_llm_structured(messages=messages, response_model=MockResponseModel)
# Verify messages were normalized and None values were stripped
assert len(received_messages) == 2
for msg in received_messages:
assert "role" in msg
assert "content" in msg
# name field should NOT be present if it was None
assert "name" not in msg or msg["name"] is not None
@pytest.mark.asyncio
async def test_none_values_stripped_from_dict_messages(self, llm_bus, service_registry):
"""Test that None values are stripped from dict messages during normalization"""
received_messages = []
class InspectingLLMService(MockLLMService):
async def call_llm_structured(
self,
messages: List[dict],
response_model: Type[BaseModel],
max_tokens: int = 1024,
temperature: float = 0.0,
**kwargs,
) -> Tuple[BaseModel, ResourceUsage]:
received_messages.extend(messages)
return await super().call_llm_structured(messages, response_model, max_tokens, temperature, **kwargs)
service = InspectingLLMService("Inspector")
service_registry.register_service(
service_type=ServiceType.LLM,
provider=service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock"},
)
# Create dict messages with None values
messages = [
{"role": "system", "content": "You are helpful", "name": None}, # name is explicitly None
{"role": "user", "content": "Test", "name": None},
]
# Make the call
await llm_bus.call_llm_structured(messages=messages, response_model=MockResponseModel)
# Verify None values were stripped
assert len(received_messages) == 2
for msg in received_messages:
assert "role" in msg
assert "content" in msg
assert "name" not in msg # None values should be stripped
class TestEdgeCases:
"""Test edge cases and error conditions"""
@pytest.mark.asyncio
async def test_capability_filtering(self, llm_bus, service_registry):
"""Test that services are filtered by capabilities"""
# Register service without required capability
limited_service = MockLLMService("Limited")
limited_service.capabilities = ["other_capability"]
# Override get_capabilities to return limited capabilities
def limited_get_capabilities():
class Capabilities:
supports_operation_list = ["other_capability"]
return Capabilities()
limited_service.get_capabilities = limited_get_capabilities
service_registry.register_service(
service_type=ServiceType.LLM,
provider=limited_service,
capabilities=["other_capability"],
metadata={"provider": "mock"},
)
# Should fail as no service has call_llm_structured capability
with pytest.raises(RuntimeError, match="No LLM services available"):
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test"}], response_model=MockResponseModel
)
@pytest.mark.asyncio
async def test_get_available_models(self, llm_bus, service_registry):
"""Test getting available models from services"""
# Register service
service = MockLLMService("ModelProvider")
service_registry.register_service(
service_type=ServiceType.LLM, provider=service, capabilities=["get_available_models"]
)
# Get models
models = await llm_bus.get_available_models()
assert len(models) == 2
assert "ModelProvider-model-1" in models
assert "ModelProvider-model-2" in models
@pytest.mark.asyncio
async def test_bus_lifecycle(self, llm_bus):
"""Test bus start/stop lifecycle"""
# Start the bus
await llm_bus.start()
assert llm_bus._running
# Stop the bus
await llm_bus.stop()
assert not llm_bus._running
# Start again
await llm_bus.start()
assert llm_bus._running
# Double start should be safe
await llm_bus.start()
assert llm_bus._running
# Clean stop
await llm_bus.stop()
@pytest.mark.asyncio
async def test_security_mock_real_mixing(self, service_registry, time_service, telemetry_service):
"""Test that mixing mock and real LLM services is prevented"""
bus = LLMBus(service_registry=service_registry, time_service=time_service, telemetry_service=telemetry_service)
# Register mock service
mock_service = MockLLMService("MockLLM")
service_registry.register_service(
service_type=ServiceType.LLM, provider=mock_service, metadata={"provider": "mock"}
)
# Try to register real service - should fail
# Create a class without "Mock" in the name
class RealLLMService:
def __init__(self, name):
self.name = name
real_service = RealLLMService("RealLLM")
with pytest.raises(RuntimeError, match="SECURITY VIOLATION"):
service_registry.register_service(
service_type=ServiceType.LLM, provider=real_service, metadata={"provider": "openai"}
)
class TestServiceUnavailableFailover:
"""Test failover specifically for 503 Service Unavailable scenarios."""
@pytest.mark.asyncio
async def test_503_error_triggers_failover_to_secondary_provider(self, llm_bus, service_registry):
"""Test that 503 Service Unavailable from primary triggers immediate failover to secondary."""
# Create primary service that fails with 503-like errors
class ServiceUnavailableLLMService(MockLLMService):
def __init__(self, name):
super().__init__(name)
self.should_fail = True
async def call_llm_structured(self, messages, response_model, max_tokens=1024, temperature=0.0, **kwargs):
self.call_count += 1 # Track calls even if they fail
if self.should_fail:
# Simulate the same type of error that would cause circuit breaker activation
raise RuntimeError("LLM service unavailable - circuit breaker activated for failover")
return await super().call_llm_structured(messages, response_model, max_tokens, temperature, **kwargs)
# Create secondary service that always succeeds
primary_service = ServiceUnavailableLLMService("Together.AI")
secondary_service = MockLLMService("Lambda.AI")
# Register primary with high priority - mark as mock to avoid security conflicts
service_registry.register_service(
service_type=ServiceType.LLM,
provider=primary_service,
priority=Priority.HIGH,
capabilities=["call_llm_structured"],
metadata={"provider": "mock", "domain": "general", "original_provider": "together"},
)
# Register secondary with normal priority - mark as mock to avoid security conflicts
service_registry.register_service(
service_type=ServiceType.LLM,
provider=secondary_service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock", "domain": "general", "original_provider": "lambda"},
)
# First call should fail on primary, succeed on secondary
result, usage = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test failover"}], response_model=MockResponseModel
)
# Should get response from secondary service
assert result.message == "Response from Lambda.AI"
assert primary_service.call_count == 1 # Primary tried once per retry config (1 retry)
assert secondary_service.call_count == 1 # Secondary provided the response
# Trigger enough failures to open circuit breaker (default threshold is 5)
primary_service.call_count = 0
secondary_service.call_count = 0
# Make 4 more calls to reach the failure threshold
for i in range(4):
try:
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": f"Trigger failure {i}"}], response_model=MockResponseModel
)
except:
pass # Expected to succeed on secondary after primary fails
# Verify circuit breaker opened for primary service
primary_service_name = f"ServiceUnavailableLLMService_{id(primary_service)}"
assert primary_service_name in llm_bus.circuit_breakers
# Reset counts and test circuit breaker behavior
primary_service.call_count = 0
secondary_service.call_count = 0
result2, usage2 = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test circuit breaker"}], response_model=MockResponseModel
)
assert result2.message == "Response from Lambda.AI"
assert primary_service.call_count == 0 # Primary was skipped due to circuit breaker
assert secondary_service.call_count == 1 # Only secondary was called
@pytest.mark.asyncio
async def test_both_providers_503_error_propagates(self, llm_bus, service_registry):
"""Test that when both providers fail with 503, the error is properly propagated."""
class FailingLLMService(MockLLMService):
async def call_llm_structured(self, messages, response_model, max_tokens=1024, temperature=0.0, **kwargs):
self.call_count += 1 # Track calls even if they fail
raise RuntimeError("LLM service unavailable - circuit breaker activated for failover")
# Register two failing services
primary_service = FailingLLMService("Together.AI")
secondary_service = FailingLLMService("Lambda.AI")
service_registry.register_service(
service_type=ServiceType.LLM,
provider=primary_service,
priority=Priority.HIGH,
capabilities=["call_llm_structured"],
metadata={"provider": "mock", "original_provider": "together"},
)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=secondary_service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock", "original_provider": "lambda"},
)
# Should fail with appropriate error after trying both providers
with pytest.raises(RuntimeError) as exc_info:
await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Test all failing"}], response_model=MockResponseModel
)
# Should indicate all services failed
assert "All LLM services failed" in str(exc_info.value)
# Both services should have been tried (1 retry each per config)
assert primary_service.call_count == 1
assert secondary_service.call_count == 1
@pytest.mark.asyncio
async def test_primary_recovery_after_503_failover(self, llm_bus, service_registry, time_service):
"""Test that primary service can recover after 503 errors and circuit breaker recovery."""
class RecoverableLLMService(MockLLMService):
def __init__(self, name):
super().__init__(name)
self.should_fail = True
async def call_llm_structured(self, messages, response_model, max_tokens=1024, temperature=0.0, **kwargs):
if self.should_fail:
self.call_count += 1 # Track calls that fail
raise RuntimeError("LLM service unavailable - circuit breaker activated for failover")
return await super().call_llm_structured(messages, response_model, max_tokens, temperature, **kwargs)
primary_service = RecoverableLLMService("Together.AI")
secondary_service = MockLLMService("Lambda.AI")
# Register services - mark as mock to avoid security conflicts
service_registry.register_service(
service_type=ServiceType.LLM,
provider=primary_service,
priority=Priority.HIGH,
capabilities=["call_llm_structured"],
metadata={"provider": "mock", "original_provider": "together"},
)
service_registry.register_service(
service_type=ServiceType.LLM,
provider=secondary_service,
priority=Priority.NORMAL,
capabilities=["call_llm_structured"],
metadata={"provider": "mock", "original_provider": "lambda"},
)
# Initial call fails over to secondary
result1, _ = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Initial call"}], response_model=MockResponseModel
)
assert result1.message == "Response from Lambda.AI"
# Primary service recovers
primary_service.should_fail = False
# Simulate circuit breaker recovery timeout
primary_service_name = f"RecoverableLLMService_{id(primary_service)}"
cb = llm_bus.circuit_breakers[primary_service_name]
# Manually trigger circuit breaker recovery for testing
cb.reset()
# Next call should try primary again and succeed
primary_service.call_count = 0
secondary_service.call_count = 0
result2, _ = await llm_bus.call_llm_structured(
messages=[{"role": "user", "content": "Recovery test"}], response_model=MockResponseModel
)
assert result2.message == "Response from Together.AI"
assert primary_service.call_count == 1 # Primary was called and succeeded
assert secondary_service.call_count == 0 # Secondary wasn't needed
"""
Comprehensive tests for LLM Bus architecture
Tests:
- Multi-provider scenarios
- Provider failover mechanisms
- Load distribution strategies (round-robin, latency-based, random, least-loaded)
- Circuit breaker functionality
- Health checks
- Service registration/deregistration
- Priority-based selection
- Thread safety and concurrent access patterns
"""
import asyncio
import random
from datetime import datetime, timedelta, timezone
from typing import List, Tuple, Type
from unittest.mock import patch
import pytest
from pydantic import BaseModel
from ciris_engine.logic.buses.llm_bus import DistributionStrategy, LLMBus
from ciris_engine.logic.registries.base import Priority, ServiceRegistry
from ciris_engine.logic.registries.circuit_breaker import CircuitState
from ciris_engine.schemas.runtime.enums import ServiceType
from ciris_engine.schemas.runtime.resources import ResourceUsage
class MockResponseModel(BaseModel):
"""Mock response model for testing (not a test class - renamed to avoid pytest collection)"""
message: str
confidence: float = 1.0
class MockTimeService:
"""Mock time service for testing"""
def __init__(self):
self.current_time = datetime.now(timezone.utc)
self.frozen = False
def now(self) -> datetime:
if self.frozen:
return self.current_time
return datetime.now(timezone.utc)
def timestamp(self) -> float:
return self.now().timestamp()
def freeze(self, dt: datetime = None):
self.frozen = True
if dt:
self.current_time = dt
def advance(self, seconds: float):
if self.frozen:
self.current_time += timedelta(seconds=seconds)
class MockLLMService:
"""Mock LLM service for testing"""
def __init__(self, name: str, latency_ms: float = 100, failure_rate: float = 0.0):
self.name = name
self.latency_ms = latency_ms
self.failure_rate = failure_rate
self.call_count = 0
self.healthy = True
self.capabilities = ["call_llm_structured", "get_available_models"]
async def call_llm_structured(
self,
messages: List[dict],
response_model: Type[BaseModel],
max_tokens: int = 1024,
temperature: float = 0.0,
**kwargs,
) -> Tuple[BaseModel, ResourceUsage]:
"""Simulate LLM call with configurable latency and failure"""
self.call_count += 1
# Simulate latency
await asyncio.sleep(self.latency_ms / 1000)
# Simulate failures
if random.random() < self.failure_rate:
raise RuntimeError(f"{self.name} simulated failure")
# Return test response
response = MockResponseModel(message=f"Response from {self.name}")
usage = ResourceUsage(
tokens_used=100,
tokens_input=50,
tokens_output=50,
cost_cents=0.01,
model_used=f"{self.name}-model",
carbon_grams=0.001,
Xet Storage Details
- Size:
- 41.7 kB
- Xet hash:
- 81959ddb0fdcdc5ba30894a89c7569f0adf727ae0f073e889773b08342f42425
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.