Spaces:
Sleeping
Sleeping
File size: 8,105 Bytes
507973e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | """
Integration tests for multi-provider failover.
Tests automatic failover between OpenAI and Gemini providers,
conversation context preservation, and error handling.
Task Reference: T016
Spec Reference: specs/001-multi-model-chat/spec.md (US2)
"""
import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from src.api.chat import provider_router
from src.services.agent_service import todo_agent
class TestProviderFailover:
"""Tests for automatic provider failover."""
@pytest.mark.asyncio
async def test_openai_failure_fallback_to_gemini(self):
"""Test that OpenAI failure triggers Gemini fallback."""
# Mock OpenAI to fail, Gemini to succeed
with patch("src.services.router.Runner.run") as mock_run:
# First call (OpenAI) raises exception
# Second call (Gemini) succeeds
mock_result = MagicMock()
mock_result.final_output = "Response from Gemini"
mock_result.tool_calls = []
mock_run.side_effect = [
Exception("OpenAI API error"),
mock_result,
]
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Test message",
preferred_provider="openai",
)
# Should fallback to gemini
assert provider_used == "gemini"
assert result.final_output == "Response from Gemini"
assert mock_run.call_count == 2
@pytest.mark.asyncio
async def test_gemini_failure_fallback_to_openai(self):
"""Test that Gemini failure triggers OpenAI fallback."""
# Mock Gemini to fail, OpenAI to succeed
with patch("src.services.router.Runner.run") as mock_run:
# First call (Gemini) raises exception
# Second call (OpenAI) succeeds
mock_result = MagicMock()
mock_result.final_output = "Response from OpenAI"
mock_result.tool_calls = []
mock_run.side_effect = [
Exception("Gemini API error"),
mock_result,
]
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Test message",
preferred_provider="gemini",
)
# Should fallback to openai
assert provider_used == "openai"
assert result.final_output == "Response from OpenAI"
assert mock_run.call_count == 2
@pytest.mark.asyncio
async def test_both_providers_fail_returns_503(self):
"""Test that both providers failing raises HTTPException 503."""
from fastapi import HTTPException
# Mock both providers to fail
with patch("src.services.router.Runner.run") as mock_run:
mock_run.side_effect = [
Exception("OpenAI API error"),
Exception("Gemini API error"),
]
with pytest.raises(HTTPException) as exc_info:
await provider_router.run_with_failover(
agent=todo_agent,
user_input="Test message",
preferred_provider="openai",
)
assert exc_info.value.status_code == 503
assert "All AI providers are currently unavailable" in str(
exc_info.value.detail
)
assert mock_run.call_count == 2
class TestConversationContextPreservation:
"""Tests for conversation context preservation across failover."""
@pytest.mark.asyncio
async def test_context_preserved_across_failover(self):
"""Test that conversation context is preserved when provider fails over."""
# This test verifies that the agent's context (conversation history)
# is not lost when switching providers mid-conversation
with patch("src.services.router.Runner.run") as mock_run:
# First message: OpenAI succeeds
first_result = MagicMock()
first_result.final_output = "Task added"
first_result.tool_calls = ["add_todo"]
# Second message: OpenAI fails, Gemini succeeds
second_result = MagicMock()
second_result.final_output = "Task completed"
second_result.tool_calls = ["complete_todo"]
mock_run.side_effect = [
first_result, # First message (OpenAI)
Exception("OpenAI timeout"), # Second message primary attempt
second_result, # Second message failover (Gemini)
]
# First message (OpenAI)
result1, provider1 = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Add a task: Buy groceries",
preferred_provider="openai",
)
assert provider1 == "openai"
assert result1.final_output == "Task added"
# Second message (should failover to Gemini but preserve context)
result2, provider2 = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Mark it as complete",
preferred_provider="openai",
)
assert provider2 == "gemini"
assert result2.final_output == "Task completed"
# Verify run was called 3 times total (1 success + 1 fail + 1 success)
assert mock_run.call_count == 3
class TestToolCallsWithFailover:
"""Tests for tool calls working correctly with both providers."""
@pytest.mark.asyncio
async def test_tool_calls_work_with_openai(self):
"""Test that tool calls work when using OpenAI."""
with patch("src.services.router.Runner.run") as mock_run:
mock_result = MagicMock()
mock_result.final_output = "I've added the task"
mock_result.tool_calls = ["add_todo"]
mock_run.return_value = mock_result
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Add a task: Test task",
preferred_provider="openai",
)
assert provider_used == "openai"
assert "add_todo" in result.tool_calls
@pytest.mark.asyncio
async def test_tool_calls_work_with_gemini(self):
"""Test that tool calls work when using Gemini."""
with patch("src.services.router.Runner.run") as mock_run:
mock_result = MagicMock()
mock_result.final_output = "I've listed your tasks"
mock_result.tool_calls = ["list_todos"]
mock_run.return_value = mock_result
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Show my tasks",
preferred_provider="gemini",
)
assert provider_used == "gemini"
assert "list_todos" in result.tool_calls
@pytest.mark.asyncio
async def test_tool_calls_preserved_during_failover(self):
"""Test that tool calls work correctly after failover."""
with patch("src.services.router.Runner.run") as mock_run:
# OpenAI fails, Gemini succeeds with tool call
gemini_result = MagicMock()
gemini_result.final_output = "Task deleted"
gemini_result.tool_calls = ["delete_todo"]
mock_run.side_effect = [
Exception("OpenAI rate limit"),
gemini_result,
]
result, provider_used = await provider_router.run_with_failover(
agent=todo_agent,
user_input="Delete the first task",
preferred_provider="openai",
)
# Should failover to Gemini and still execute tool call
assert provider_used == "gemini"
assert "delete_todo" in result.tool_calls
assert result.final_output == "Task deleted"
|