""" Integration tests for multi-provider failover. Tests automatic failover between OpenAI and Gemini providers, conversation context preservation, and error handling. Task Reference: T016 Spec Reference: specs/001-multi-model-chat/spec.md (US2) """ import pytest from unittest.mock import AsyncMock, patch, MagicMock from src.api.chat import provider_router from src.services.agent_service import todo_agent class TestProviderFailover: """Tests for automatic provider failover.""" @pytest.mark.asyncio async def test_openai_failure_fallback_to_gemini(self): """Test that OpenAI failure triggers Gemini fallback.""" # Mock OpenAI to fail, Gemini to succeed with patch("src.services.router.Runner.run") as mock_run: # First call (OpenAI) raises exception # Second call (Gemini) succeeds mock_result = MagicMock() mock_result.final_output = "Response from Gemini" mock_result.tool_calls = [] mock_run.side_effect = [ Exception("OpenAI API error"), mock_result, ] result, provider_used = await provider_router.run_with_failover( agent=todo_agent, user_input="Test message", preferred_provider="openai", ) # Should fallback to gemini assert provider_used == "gemini" assert result.final_output == "Response from Gemini" assert mock_run.call_count == 2 @pytest.mark.asyncio async def test_gemini_failure_fallback_to_openai(self): """Test that Gemini failure triggers OpenAI fallback.""" # Mock Gemini to fail, OpenAI to succeed with patch("src.services.router.Runner.run") as mock_run: # First call (Gemini) raises exception # Second call (OpenAI) succeeds mock_result = MagicMock() mock_result.final_output = "Response from OpenAI" mock_result.tool_calls = [] mock_run.side_effect = [ Exception("Gemini API error"), mock_result, ] result, provider_used = await provider_router.run_with_failover( agent=todo_agent, user_input="Test message", preferred_provider="gemini", ) # Should fallback to openai assert provider_used == "openai" assert result.final_output == "Response from OpenAI" assert mock_run.call_count == 2 @pytest.mark.asyncio async def test_both_providers_fail_returns_503(self): """Test that both providers failing raises HTTPException 503.""" from fastapi import HTTPException # Mock both providers to fail with patch("src.services.router.Runner.run") as mock_run: mock_run.side_effect = [ Exception("OpenAI API error"), Exception("Gemini API error"), ] with pytest.raises(HTTPException) as exc_info: await provider_router.run_with_failover( agent=todo_agent, user_input="Test message", preferred_provider="openai", ) assert exc_info.value.status_code == 503 assert "All AI providers are currently unavailable" in str( exc_info.value.detail ) assert mock_run.call_count == 2 class TestConversationContextPreservation: """Tests for conversation context preservation across failover.""" @pytest.mark.asyncio async def test_context_preserved_across_failover(self): """Test that conversation context is preserved when provider fails over.""" # This test verifies that the agent's context (conversation history) # is not lost when switching providers mid-conversation with patch("src.services.router.Runner.run") as mock_run: # First message: OpenAI succeeds first_result = MagicMock() first_result.final_output = "Task added" first_result.tool_calls = ["add_todo"] # Second message: OpenAI fails, Gemini succeeds second_result = MagicMock() second_result.final_output = "Task completed" second_result.tool_calls = ["complete_todo"] mock_run.side_effect = [ first_result, # First message (OpenAI) Exception("OpenAI timeout"), # Second message primary attempt second_result, # Second message failover (Gemini) ] # First message (OpenAI) result1, provider1 = await provider_router.run_with_failover( agent=todo_agent, user_input="Add a task: Buy groceries", preferred_provider="openai", ) assert provider1 == "openai" assert result1.final_output == "Task added" # Second message (should failover to Gemini but preserve context) result2, provider2 = await provider_router.run_with_failover( agent=todo_agent, user_input="Mark it as complete", preferred_provider="openai", ) assert provider2 == "gemini" assert result2.final_output == "Task completed" # Verify run was called 3 times total (1 success + 1 fail + 1 success) assert mock_run.call_count == 3 class TestToolCallsWithFailover: """Tests for tool calls working correctly with both providers.""" @pytest.mark.asyncio async def test_tool_calls_work_with_openai(self): """Test that tool calls work when using OpenAI.""" with patch("src.services.router.Runner.run") as mock_run: mock_result = MagicMock() mock_result.final_output = "I've added the task" mock_result.tool_calls = ["add_todo"] mock_run.return_value = mock_result result, provider_used = await provider_router.run_with_failover( agent=todo_agent, user_input="Add a task: Test task", preferred_provider="openai", ) assert provider_used == "openai" assert "add_todo" in result.tool_calls @pytest.mark.asyncio async def test_tool_calls_work_with_gemini(self): """Test that tool calls work when using Gemini.""" with patch("src.services.router.Runner.run") as mock_run: mock_result = MagicMock() mock_result.final_output = "I've listed your tasks" mock_result.tool_calls = ["list_todos"] mock_run.return_value = mock_result result, provider_used = await provider_router.run_with_failover( agent=todo_agent, user_input="Show my tasks", preferred_provider="gemini", ) assert provider_used == "gemini" assert "list_todos" in result.tool_calls @pytest.mark.asyncio async def test_tool_calls_preserved_during_failover(self): """Test that tool calls work correctly after failover.""" with patch("src.services.router.Runner.run") as mock_run: # OpenAI fails, Gemini succeeds with tool call gemini_result = MagicMock() gemini_result.final_output = "Task deleted" gemini_result.tool_calls = ["delete_todo"] mock_run.side_effect = [ Exception("OpenAI rate limit"), gemini_result, ] result, provider_used = await provider_router.run_with_failover( agent=todo_agent, user_input="Delete the first task", preferred_provider="openai", ) # Should failover to Gemini and still execute tool call assert provider_used == "gemini" assert "delete_todo" in result.tool_calls assert result.final_output == "Task deleted"