ai-api-testing / backend /tests /test_llm_planner.py
Amitkasar's picture
Add multi-LLM support, settings & migrations
1e24842
Raw History Blame Contribute Delete
2.19 kB
import json
from unittest.mock import MagicMock
import pytest
from backend.services.llm_planner import generate_execution_plan
_VALID_RESPONSE = {
"execution_plan": [
{"step": 1, "endpoint": "POST /auth/login", "purpose": "Authenticate the user"},
{"step": 2, "endpoint": "GET /profile", "purpose": "Fetch user profile"},
],
"suggested_credentials": {"username": "tester@example.com", "password": "Pass123!"},
}
_TOKEN_USAGE = {"prompt_tokens": 120, "completion_tokens": 60, "total_tokens": 180}
def _mock_llm(return_value=None, side_effect=None):
m = MagicMock()
if side_effect is not None:
m.invoke.side_effect = side_effect
else:
m.invoke.return_value = return_value
return m
def test_returns_execution_plan():
llm = _mock_llm((json.dumps(_VALID_RESPONSE), _TOKEN_USAGE))
result = generate_execution_plan(
parsed_spec={"endpoints": []},
base_url="https://api.example.com",
scenario="Login and view profile",
llm=llm,
)
assert len(result.execution_plan) == 2
assert result.execution_plan[0]["step"] == 1
assert result.suggested_credentials["username"] == "tester@example.com"
def test_token_usage_is_passed_through():
llm = _mock_llm((json.dumps(_VALID_RESPONSE), _TOKEN_USAGE))
result = generate_execution_plan(
parsed_spec={}, base_url="https://api.example.com", scenario="Login", llm=llm
)
assert result.token_usage == _TOKEN_USAGE
def test_retries_once_on_parse_failure():
llm = _mock_llm(side_effect=[
("not valid json {{{{", None),
(json.dumps(_VALID_RESPONSE), _TOKEN_USAGE),
])
result = generate_execution_plan(
parsed_spec={}, base_url="https://api.example.com", scenario="Login", llm=llm
)
assert llm.invoke.call_count == 2
assert len(result.execution_plan) == 2
def test_raises_after_two_parse_failures():
llm = _mock_llm(side_effect=[
("bad json", None),
("still bad json", None),
])
with pytest.raises(Exception):
generate_execution_plan(
parsed_spec={}, base_url="https://api.example.com", scenario="Login", llm=llm
)