ai-api-testing / backend /services /chat_assistant.py
Amitkasar's picture
Add multi-LLM support, settings & migrations
1e24842
Raw History Blame Contribute Delete
7.39 kB
"""
chat_assistant.py
─────────────────
AI brain for the Chat Assistant. Takes a user's natural-language message and the
full testing context, then returns a structured JSON action so the frontend knows
what to do (execute a step, explain an error, modify a request, generate tests, etc.).
"""
import json
import re
from typing import Any, Dict, List, Optional
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from backend.core.logging import get_logger
from backend.integrations.llm.base import BaseLLMClient
logger = get_logger(__name__)
SYSTEM_PROMPT = """You are an AI assistant integrated into an API testing tool (similar to Postman AI).
Your job is to help the user control their API testing workflow through natural language.
You have access to the following context:
- Active API Spec: the spec file being tested
- Execution Plan: the ordered list of test steps the AI generated
- Session Context Variables: dynamic values extracted from previous steps (tokens, IDs, etc.)
- Last Step Result: the most recent HTTP request/response + AI analysis
- All Step Results: execution history for all steps in the current session
IMPORTANT: You must ALWAYS respond with a valid JSON object. No markdown, no explanation outside JSON.
JSON response format:
{
"action": "<one of the actions below>",
"message": "<friendly message to show in the chat bubble β€” always required>",
"step": <step number, integer β€” only for step-specific actions>,
"changes": { "body": {"field": "value"}, "query_params": {...} }, // only for modify_request, MUST specify section
"assertions": ["check status is 200", "check response has token field"], // only for generate_tests
"explanation": "<detailed explanation text>" // only for explain action
}
Available actions:
- "execute_step" β†’ user wants to run a specific step (provide "step" number)
- "run_all" β†’ user wants to run all steps in sequence
- "skip_step" β†’ user wants to skip a step (provide "step" number)
- "retry_step" β†’ user wants to retry the last or specific step (provide "step" number)
- "modify_request" β†’ user wants to modify request params before executing. You MUST provide "step" (the integer step to modify) and "changes" (dict of what to change grouped by body/query_params/headers).
- "explain" β†’ user wants an explanation of an error or result (provide "explanation")
- "generate_tests" β†’ user wants test assertions generated (provide "assertions" list)
- "reply" β†’ general answer, no action triggered
Rules:
1. Always include "message" β€” it appears in the chat bubble.
2. For "explain" action, also fill "explanation" with a detailed, technical breakdown.
3. For "generate_tests", make assertions practical and specific to the API.
4. For "modify_request", structure "changes" with base keys `body`, `query_params`, `path_params`, or `headers`. Use `{{variable_name}}` syntax if the user asks to use a context variable. Provide the `step` number to modify.
5. If you're unsure what the user wants, use "reply" action and ask for clarification.
6. Be concise in "message" (1-3 sentences). Use "explanation" for long text.
"""
def _build_context_block(
spec_name: Optional[str],
execution_plan: List[Dict],
context_variables: Dict,
last_step_result: Optional[Dict],
all_step_results: Optional[Dict] = None,
) -> str:
"""Build a compact context string to inject into the system message."""
parts = []
if spec_name:
parts.append(f"Active Spec: {spec_name}")
if execution_plan:
plan_lines = [f" Step {s.get('step', i+1)}: {s.get('endpoint', '')} β€” {s.get('purpose', '')}"
for i, s in enumerate(execution_plan)]
parts.append("Execution Plan:\n" + "\n".join(plan_lines))
else:
parts.append("Execution Plan: (none β€” no plan generated yet)")
if context_variables:
ctx_str = json.dumps(context_variables, separators=(',', ':'))
parts.append(f"Session Context Variables: {ctx_str}")
else:
parts.append("Session Context Variables: (empty β€” no steps executed yet)")
if last_step_result:
step_def = last_step_result.get("step_definition", {})
response = last_step_result.get("response", {})
analysis = last_step_result.get("analysis", {})
parts.append(
f"Last Step Result:\n"
f" Endpoint: {step_def.get('endpoint', 'unknown')}\n"
f" Status Code: {response.get('status_code', 'unknown')}\n"
f" AI Analysis: {analysis.get('status', 'unknown')} β€” {analysis.get('response_summary', '')}\n"
f" Response Body (truncated): {str(response.get('body', ''))[:500]}"
)
else:
parts.append("Last Step Result: (no step has been executed yet)")
if all_step_results:
all_res_str = []
for step_num, s in all_step_results.items():
sc = s.get("response", {}).get("status_code", "N/A")
all_res_str.append(f" Step {step_num}: Status {sc}")
parts.append("All Step Results Summary:\n" + "\n".join(all_res_str))
return "\n\n".join(parts)
def _build_history_messages(chat_history: List[Dict]) -> List:
"""Convert DB chat message dicts to LangChain message objects (last 20)."""
messages = []
for msg in chat_history[-20:]:
role = msg.get("role", "user")
content = msg.get("content", "")
if role == "user":
messages.append(HumanMessage(content=content))
else:
messages.append(AIMessage(content=content))
return messages
def process_message(
message: str,
spec_name: Optional[str],
execution_plan: List[Dict],
context_variables: Dict,
last_step_result: Optional[Dict],
all_step_results: Optional[Dict],
chat_history: List[Dict],
llm: BaseLLMClient = None,
) -> Dict[str, Any]:
"""
Main entry point. Sends the user message + full context to Mistral and returns
a parsed action dict. Falls back to a plain 'reply' action if parsing fails.
"""
context_block = _build_context_block(spec_name, execution_plan, context_variables, last_step_result, all_step_results)
full_system = SYSTEM_PROMPT + f"\n\n--- CURRENT CONTEXT ---\n{context_block}\n--- END CONTEXT ---"
messages = [SystemMessage(content=full_system)]
messages.extend(_build_history_messages(chat_history))
messages.append(HumanMessage(content=message))
try:
content, token_usage = llm.invoke(messages)
raw = content.strip()
# Strip markdown code fences if present
raw = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.MULTILINE)
raw = re.sub(r"\s*```$", "", raw, flags=re.MULTILINE)
parsed = json.loads(raw)
# Ensure required fields
if "action" not in parsed:
parsed["action"] = "reply"
if "message" not in parsed:
parsed["message"] = raw
parsed["_token_usage"] = token_usage
return parsed
except Exception as e:
logger.error("Chat AI error: %s", e)
return {
"action": "reply",
"message": "I encountered an error processing your request. Please try again.",
"explanation": str(e),
"_token_usage": None,
}