tasklert-api / tests /integration /test_failover.py
hashir-qt's picture
initial deploy
507973e
Raw History Blame Contribute Delete
8.11 kB
"""
Integration tests for multi-provider failover.
Tests automatic failover between OpenAI and Gemini providers,
conversation context preservation, and error handling.
Task Reference: T016
Spec Reference: specs/001-multi-model-chat/spec.md (US2)
"""
import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from src.api.chat import provider_router
from src.services.agent_service import todo_agent
class TestProviderFailover:
"""Tests for automatic provider failover."""
@pytest.mark.asyncio
async def test_openai_failure_fallback_to_gemini(self):
"""Test that OpenAI failure triggers Gemini fallback."""
# Mock OpenAI to fail, Gemini to succeed
with patch("src.services.router.Runner.run") as mock_run:
# First call (OpenAI) raises exception
# Second call (Gemini) succeeds
mock_result = MagicMock()
mock_result.final_output = "Response from Gemini"
mock_result.tool_calls = []
mock_run.side_effect = [
Exception("OpenAI API error"),
mock_result,
]
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Test message",
preferred_provider="openai",
)
# Should fallback to gemini
assert provider_used == "gemini"
assert result.final_output == "Response from Gemini"
assert mock_run.call_count == 2
@pytest.mark.asyncio
async def test_gemini_failure_fallback_to_openai(self):
"""Test that Gemini failure triggers OpenAI fallback."""
# Mock Gemini to fail, OpenAI to succeed
with patch("src.services.router.Runner.run") as mock_run:
# First call (Gemini) raises exception
# Second call (OpenAI) succeeds
mock_result = MagicMock()
mock_result.final_output = "Response from OpenAI"
mock_result.tool_calls = []
mock_run.side_effect = [
Exception("Gemini API error"),
mock_result,
]
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Test message",
preferred_provider="gemini",
)
# Should fallback to openai
assert provider_used == "openai"
assert result.final_output == "Response from OpenAI"
assert mock_run.call_count == 2
@pytest.mark.asyncio
async def test_both_providers_fail_returns_503(self):
"""Test that both providers failing raises HTTPException 503."""
from fastapi import HTTPException
# Mock both providers to fail
with patch("src.services.router.Runner.run") as mock_run:
mock_run.side_effect = [
Exception("OpenAI API error"),
Exception("Gemini API error"),
]
with pytest.raises(HTTPException) as exc_info:
await provider_router.run_with_failover(
agent=todo_agent,
user_input="Test message",
preferred_provider="openai",
)
assert exc_info.value.status_code == 503
assert "All AI providers are currently unavailable" in str(
exc_info.value.detail
)
assert mock_run.call_count == 2
class TestConversationContextPreservation:
"""Tests for conversation context preservation across failover."""
@pytest.mark.asyncio
async def test_context_preserved_across_failover(self):
"""Test that conversation context is preserved when provider fails over."""
# This test verifies that the agent's context (conversation history)
# is not lost when switching providers mid-conversation
with patch("src.services.router.Runner.run") as mock_run:
# First message: OpenAI succeeds
first_result = MagicMock()
first_result.final_output = "Task added"
first_result.tool_calls = ["add_todo"]
# Second message: OpenAI fails, Gemini succeeds
second_result = MagicMock()
second_result.final_output = "Task completed"
second_result.tool_calls = ["complete_todo"]
mock_run.side_effect = [
first_result, # First message (OpenAI)
Exception("OpenAI timeout"), # Second message primary attempt
second_result, # Second message failover (Gemini)
]
# First message (OpenAI)
result1, provider1 = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Add a task: Buy groceries",
preferred_provider="openai",
)
assert provider1 == "openai"
assert result1.final_output == "Task added"
# Second message (should failover to Gemini but preserve context)
result2, provider2 = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Mark it as complete",
preferred_provider="openai",
)
assert provider2 == "gemini"
assert result2.final_output == "Task completed"
# Verify run was called 3 times total (1 success + 1 fail + 1 success)
assert mock_run.call_count == 3
class TestToolCallsWithFailover:
"""Tests for tool calls working correctly with both providers."""
@pytest.mark.asyncio
async def test_tool_calls_work_with_openai(self):
"""Test that tool calls work when using OpenAI."""
with patch("src.services.router.Runner.run") as mock_run:
mock_result = MagicMock()
mock_result.final_output = "I've added the task"
mock_result.tool_calls = ["add_todo"]
mock_run.return_value = mock_result
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Add a task: Test task",
preferred_provider="openai",
)
assert provider_used == "openai"
assert "add_todo" in result.tool_calls
@pytest.mark.asyncio
async def test_tool_calls_work_with_gemini(self):
"""Test that tool calls work when using Gemini."""
with patch("src.services.router.Runner.run") as mock_run:
mock_result = MagicMock()
mock_result.final_output = "I've listed your tasks"
mock_result.tool_calls = ["list_todos"]
mock_run.return_value = mock_result
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Show my tasks",
preferred_provider="gemini",
)
assert provider_used == "gemini"
assert "list_todos" in result.tool_calls
@pytest.mark.asyncio
async def test_tool_calls_preserved_during_failover(self):
"""Test that tool calls work correctly after failover."""
with patch("src.services.router.Runner.run") as mock_run:
# OpenAI fails, Gemini succeeds with tool call
gemini_result = MagicMock()
gemini_result.final_output = "Task deleted"
gemini_result.tool_calls = ["delete_todo"]
mock_run.side_effect = [
Exception("OpenAI rate limit"),
gemini_result,
]
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Delete the first task",
preferred_provider="openai",
)
# Should failover to Gemini and still execute tool call
assert provider_used == "gemini"
assert "delete_todo" in result.tool_calls
assert result.final_output == "Task deleted"