File size: 8,105 Bytes
507973e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
"""
Integration tests for multi-provider failover.

Tests automatic failover between OpenAI and Gemini providers,
conversation context preservation, and error handling.

Task Reference: T016
Spec Reference: specs/001-multi-model-chat/spec.md (US2)
"""

import pytest
from unittest.mock import AsyncMock, patch, MagicMock

from src.api.chat import provider_router
from src.services.agent_service import todo_agent


class TestProviderFailover:
    """Tests for automatic provider failover."""

    @pytest.mark.asyncio
    async def test_openai_failure_fallback_to_gemini(self):
        """Test that OpenAI failure triggers Gemini fallback."""
        # Mock OpenAI to fail, Gemini to succeed
        with patch("src.services.router.Runner.run") as mock_run:
            # First call (OpenAI) raises exception
            # Second call (Gemini) succeeds
            mock_result = MagicMock()
            mock_result.final_output = "Response from Gemini"
            mock_result.tool_calls = []

            mock_run.side_effect = [
                Exception("OpenAI API error"),
                mock_result,
            ]

            result, provider_used = await provider_router.run_with_failover(
                agent=todo_agent,
                user_input="Test message",
                preferred_provider="openai",
            )

            # Should fallback to gemini
            assert provider_used == "gemini"
            assert result.final_output == "Response from Gemini"
            assert mock_run.call_count == 2

    @pytest.mark.asyncio
    async def test_gemini_failure_fallback_to_openai(self):
        """Test that Gemini failure triggers OpenAI fallback."""
        # Mock Gemini to fail, OpenAI to succeed
        with patch("src.services.router.Runner.run") as mock_run:
            # First call (Gemini) raises exception
            # Second call (OpenAI) succeeds
            mock_result = MagicMock()
            mock_result.final_output = "Response from OpenAI"
            mock_result.tool_calls = []

            mock_run.side_effect = [
                Exception("Gemini API error"),
                mock_result,
            ]

            result, provider_used = await provider_router.run_with_failover(
                agent=todo_agent,
                user_input="Test message",
                preferred_provider="gemini",
            )

            # Should fallback to openai
            assert provider_used == "openai"
            assert result.final_output == "Response from OpenAI"
            assert mock_run.call_count == 2

    @pytest.mark.asyncio
    async def test_both_providers_fail_returns_503(self):
        """Test that both providers failing raises HTTPException 503."""
        from fastapi import HTTPException

        # Mock both providers to fail
        with patch("src.services.router.Runner.run") as mock_run:
            mock_run.side_effect = [
                Exception("OpenAI API error"),
                Exception("Gemini API error"),
            ]

            with pytest.raises(HTTPException) as exc_info:
                await provider_router.run_with_failover(
                    agent=todo_agent,
                    user_input="Test message",
                    preferred_provider="openai",
                )

            assert exc_info.value.status_code == 503
            assert "All AI providers are currently unavailable" in str(
                exc_info.value.detail
            )
            assert mock_run.call_count == 2


class TestConversationContextPreservation:
    """Tests for conversation context preservation across failover."""

    @pytest.mark.asyncio
    async def test_context_preserved_across_failover(self):
        """Test that conversation context is preserved when provider fails over."""
        # This test verifies that the agent's context (conversation history)
        # is not lost when switching providers mid-conversation

        with patch("src.services.router.Runner.run") as mock_run:
            # First message: OpenAI succeeds
            first_result = MagicMock()
            first_result.final_output = "Task added"
            first_result.tool_calls = ["add_todo"]

            # Second message: OpenAI fails, Gemini succeeds
            second_result = MagicMock()
            second_result.final_output = "Task completed"
            second_result.tool_calls = ["complete_todo"]

            mock_run.side_effect = [
                first_result,  # First message (OpenAI)
                Exception("OpenAI timeout"),  # Second message primary attempt
                second_result,  # Second message failover (Gemini)
            ]

            # First message (OpenAI)
            result1, provider1 = await provider_router.run_with_failover(
                agent=todo_agent,
                user_input="Add a task: Buy groceries",
                preferred_provider="openai",
            )

            assert provider1 == "openai"
            assert result1.final_output == "Task added"

            # Second message (should failover to Gemini but preserve context)
            result2, provider2 = await provider_router.run_with_failover(
                agent=todo_agent,
                user_input="Mark it as complete",
                preferred_provider="openai",
            )

            assert provider2 == "gemini"
            assert result2.final_output == "Task completed"

            # Verify run was called 3 times total (1 success + 1 fail + 1 success)
            assert mock_run.call_count == 3


class TestToolCallsWithFailover:
    """Tests for tool calls working correctly with both providers."""

    @pytest.mark.asyncio
    async def test_tool_calls_work_with_openai(self):
        """Test that tool calls work when using OpenAI."""
        with patch("src.services.router.Runner.run") as mock_run:
            mock_result = MagicMock()
            mock_result.final_output = "I've added the task"
            mock_result.tool_calls = ["add_todo"]

            mock_run.return_value = mock_result

            result, provider_used = await provider_router.run_with_failover(
                agent=todo_agent,
                user_input="Add a task: Test task",
                preferred_provider="openai",
            )

            assert provider_used == "openai"
            assert "add_todo" in result.tool_calls

    @pytest.mark.asyncio
    async def test_tool_calls_work_with_gemini(self):
        """Test that tool calls work when using Gemini."""
        with patch("src.services.router.Runner.run") as mock_run:
            mock_result = MagicMock()
            mock_result.final_output = "I've listed your tasks"
            mock_result.tool_calls = ["list_todos"]

            mock_run.return_value = mock_result

            result, provider_used = await provider_router.run_with_failover(
                agent=todo_agent,
                user_input="Show my tasks",
                preferred_provider="gemini",
            )

            assert provider_used == "gemini"
            assert "list_todos" in result.tool_calls

    @pytest.mark.asyncio
    async def test_tool_calls_preserved_during_failover(self):
        """Test that tool calls work correctly after failover."""
        with patch("src.services.router.Runner.run") as mock_run:
            # OpenAI fails, Gemini succeeds with tool call
            gemini_result = MagicMock()
            gemini_result.final_output = "Task deleted"
            gemini_result.tool_calls = ["delete_todo"]

            mock_run.side_effect = [
                Exception("OpenAI rate limit"),
                gemini_result,
            ]

            result, provider_used = await provider_router.run_with_failover(
                agent=todo_agent,
                user_input="Delete the first task",
                preferred_provider="openai",
            )

            # Should failover to Gemini and still execute tool call
            assert provider_used == "gemini"
            assert "delete_todo" in result.tool_calls
            assert result.final_output == "Task deleted"