Spaces:
Sleeping
Sleeping
Download tests/integration/test_failover.py from hashir-qt/tasklert-api: direct link, hf CLI and curl.
- Browser
- Download file 8.11 kB
-
https://huggingface.co/spaces/hashir-qt/tasklert-api/resolve/main/tests/integration/test_failover.py
- Command line
-
hf download hf://spaces/hashir-qt/tasklert-api/tests/integration/test_failover.py
-
curl -L -o test_failover.py https://huggingface.co/spaces/hashir-qt/tasklert-api/resolve/main/tests/integration/test_failover.py
8.11 kB
| """ | |
| Integration tests for multi-provider failover. | |
| Tests automatic failover between OpenAI and Gemini providers, | |
| conversation context preservation, and error handling. | |
| Task Reference: T016 | |
| Spec Reference: specs/001-multi-model-chat/spec.md (US2) | |
| """ | |
| import pytest | |
| from unittest.mock import AsyncMock, patch, MagicMock | |
| from src.api.chat import provider_router | |
| from src.services.agent_service import todo_agent | |
| class TestProviderFailover: | |
| """Tests for automatic provider failover.""" | |
| async def test_openai_failure_fallback_to_gemini(self): | |
| """Test that OpenAI failure triggers Gemini fallback.""" | |
| # Mock OpenAI to fail, Gemini to succeed | |
| with patch("src.services.router.Runner.run") as mock_run: | |
| # First call (OpenAI) raises exception | |
| # Second call (Gemini) succeeds | |
| mock_result = MagicMock() | |
| mock_result.final_output = "Response from Gemini" | |
| mock_result.tool_calls = [] | |
| mock_run.side_effect = [ | |
| Exception("OpenAI API error"), | |
| mock_result, | |
| ] | |
| result, provider_used = await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Test message", | |
| preferred_provider="openai", | |
| ) | |
| # Should fallback to gemini | |
| assert provider_used == "gemini" | |
| assert result.final_output == "Response from Gemini" | |
| assert mock_run.call_count == 2 | |
| async def test_gemini_failure_fallback_to_openai(self): | |
| """Test that Gemini failure triggers OpenAI fallback.""" | |
| # Mock Gemini to fail, OpenAI to succeed | |
| with patch("src.services.router.Runner.run") as mock_run: | |
| # First call (Gemini) raises exception | |
| # Second call (OpenAI) succeeds | |
| mock_result = MagicMock() | |
| mock_result.final_output = "Response from OpenAI" | |
| mock_result.tool_calls = [] | |
| mock_run.side_effect = [ | |
| Exception("Gemini API error"), | |
| mock_result, | |
| ] | |
| result, provider_used = await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Test message", | |
| preferred_provider="gemini", | |
| ) | |
| # Should fallback to openai | |
| assert provider_used == "openai" | |
| assert result.final_output == "Response from OpenAI" | |
| assert mock_run.call_count == 2 | |
| async def test_both_providers_fail_returns_503(self): | |
| """Test that both providers failing raises HTTPException 503.""" | |
| from fastapi import HTTPException | |
| # Mock both providers to fail | |
| with patch("src.services.router.Runner.run") as mock_run: | |
| mock_run.side_effect = [ | |
| Exception("OpenAI API error"), | |
| Exception("Gemini API error"), | |
| ] | |
| with pytest.raises(HTTPException) as exc_info: | |
| await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Test message", | |
| preferred_provider="openai", | |
| ) | |
| assert exc_info.value.status_code == 503 | |
| assert "All AI providers are currently unavailable" in str( | |
| exc_info.value.detail | |
| ) | |
| assert mock_run.call_count == 2 | |
| class TestConversationContextPreservation: | |
| """Tests for conversation context preservation across failover.""" | |
| async def test_context_preserved_across_failover(self): | |
| """Test that conversation context is preserved when provider fails over.""" | |
| # This test verifies that the agent's context (conversation history) | |
| # is not lost when switching providers mid-conversation | |
| with patch("src.services.router.Runner.run") as mock_run: | |
| # First message: OpenAI succeeds | |
| first_result = MagicMock() | |
| first_result.final_output = "Task added" | |
| first_result.tool_calls = ["add_todo"] | |
| # Second message: OpenAI fails, Gemini succeeds | |
| second_result = MagicMock() | |
| second_result.final_output = "Task completed" | |
| second_result.tool_calls = ["complete_todo"] | |
| mock_run.side_effect = [ | |
| first_result, # First message (OpenAI) | |
| Exception("OpenAI timeout"), # Second message primary attempt | |
| second_result, # Second message failover (Gemini) | |
| ] | |
| # First message (OpenAI) | |
| result1, provider1 = await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Add a task: Buy groceries", | |
| preferred_provider="openai", | |
| ) | |
| assert provider1 == "openai" | |
| assert result1.final_output == "Task added" | |
| # Second message (should failover to Gemini but preserve context) | |
| result2, provider2 = await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Mark it as complete", | |
| preferred_provider="openai", | |
| ) | |
| assert provider2 == "gemini" | |
| assert result2.final_output == "Task completed" | |
| # Verify run was called 3 times total (1 success + 1 fail + 1 success) | |
| assert mock_run.call_count == 3 | |
| class TestToolCallsWithFailover: | |
| """Tests for tool calls working correctly with both providers.""" | |
| async def test_tool_calls_work_with_openai(self): | |
| """Test that tool calls work when using OpenAI.""" | |
| with patch("src.services.router.Runner.run") as mock_run: | |
| mock_result = MagicMock() | |
| mock_result.final_output = "I've added the task" | |
| mock_result.tool_calls = ["add_todo"] | |
| mock_run.return_value = mock_result | |
| result, provider_used = await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Add a task: Test task", | |
| preferred_provider="openai", | |
| ) | |
| assert provider_used == "openai" | |
| assert "add_todo" in result.tool_calls | |
| async def test_tool_calls_work_with_gemini(self): | |
| """Test that tool calls work when using Gemini.""" | |
| with patch("src.services.router.Runner.run") as mock_run: | |
| mock_result = MagicMock() | |
| mock_result.final_output = "I've listed your tasks" | |
| mock_result.tool_calls = ["list_todos"] | |
| mock_run.return_value = mock_result | |
| result, provider_used = await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Show my tasks", | |
| preferred_provider="gemini", | |
| ) | |
| assert provider_used == "gemini" | |
| assert "list_todos" in result.tool_calls | |
| async def test_tool_calls_preserved_during_failover(self): | |
| """Test that tool calls work correctly after failover.""" | |
| with patch("src.services.router.Runner.run") as mock_run: | |
| # OpenAI fails, Gemini succeeds with tool call | |
| gemini_result = MagicMock() | |
| gemini_result.final_output = "Task deleted" | |
| gemini_result.tool_calls = ["delete_todo"] | |
| mock_run.side_effect = [ | |
| Exception("OpenAI rate limit"), | |
| gemini_result, | |
| ] | |
| result, provider_used = await provider_router.run_with_failover( | |
| agent=todo_agent, | |
| user_input="Delete the first task", | |
| preferred_provider="openai", | |
| ) | |
| # Should failover to Gemini and still execute tool call | |
| assert provider_used == "gemini" | |
| assert "delete_todo" in result.tool_calls | |
| assert result.final_output == "Task deleted" | |