ai-api-testing / backend /services /response_analyzer.py
Amitkasar's picture
solved issue
f1c3c6d
Raw History Blame Contribute Delete
5.38 kB
import json
from typing import Dict, Any
from langchain_core.prompts import PromptTemplate
from langchain_core.output_parsers import JsonOutputParser
from backend.core.config import settings
from backend.core.logging import get_logger
from backend.integrations.llm.base import BaseLLMClient
from backend.schemas import AnalyzeResponse
logger = get_logger(__name__)
ANALYSIS_PROMPT = """
You are a response analyzer.
Analyze status {status_code} and body {response_body} for step {step_info}.
Identify and extract generalized variables needed for future steps.
Set 'is_otp_required':true if OTP/Verification is needed.
CRITICAL: If the response contains a LIST of options, entities, or results that the user must choose between to continue their test scenario logically (e.g., a list of available buses, shopping items, flights, seats, etc.), you MUST:
1. Set "is_selection_required" to true.
2. Provide a "selection_field_name" (e.g., "bus_id", "product_id").
3. Provide "selection_options" as a list of objects containing 'label' (human readable name) and 'value' (the ID to be saved).
4. Do NOT place the item intended for selection into 'extracted_variables' automatically if selection is required.
5. If the response body is an empty list or has no valid items, DO NOT set "is_selection_required" to true. Do not guess or hallucinate options.
RETURN JSON ONLY:
{{
"step": 1, "status": "success", "is_otp_required": false, "otp_field_name": "otp",
"is_selection_required": false, "selection_field_name": null,
"selection_options": null,
"extracted_variables": {{}}
}}
"""
def analyze_api_response(
step: int,
endpoint: str,
purpose: str,
status_code: int,
response_body: str,
llm: BaseLLMClient = None,
) -> AnalyzeResponse:
"""
Uses Mistral AI via LangChain to analyze an API response and extract dynamic variables.
"""
parser = JsonOutputParser(pydantic_object=AnalyzeResponse)
prompt = PromptTemplate(
template=ANALYSIS_PROMPT,
input_variables=["step_info", "status_code", "response_body"],
)
limit = settings.RESPONSE_TRUNCATION_LIMIT
if len(response_body) > limit:
try:
parsed_body = json.loads(response_body)
response_body = json.dumps(parsed_body, separators=(',', ':'))[:limit]
except (json.JSONDecodeError, ValueError):
half = limit // 2
response_body = response_body[:half] + "\n... [TRUNCATED] ...\n" + response_body[-half:]
# ── LOG ANALYSIS INPUTS ──────────────────────────────────────────────────
logger.info("="*70)
logger.info("[ANALYZER] β–Ά RESPONSE ANALYSIS STARTED for Step %s", step)
logger.info("[ANALYZER] Endpoint : %s %s", endpoint, purpose)
logger.info("[ANALYZER] Status Code : %s", status_code)
logger.info("[ANALYZER] Response Body (first 500 chars):\n%s", response_body[:500])
logger.info("="*70)
msg = prompt.format_prompt(
step_info=f"Step {step}: {endpoint} - {purpose}",
status_code=str(status_code),
response_body=response_body
).to_messages()
logger.info("[ANALYZER] ↑ Sending response to LLM for analysis...")
content, token_usage = llm.invoke(msg)
logger.info("[ANALYZER] ↓ Raw LLM response:")
logger.info("-"*70)
logger.info("%s", content)
logger.info("-"*70)
try:
parsed_response = parser.parse(content)
except Exception as parse_err:
logger.warning("[ANALYZER] ⚠ Analysis parse failed, retrying: %s", parse_err)
try:
content, token_usage = llm.invoke(msg)
parsed_response = parser.parse(content)
except Exception as retry_err:
logger.error("[ANALYZER] βœ— Analysis failed after retry, returning safe fallback: %s", retry_err)
return AnalyzeResponse(
step=step,
status="unknown",
response_summary="AI analysis unavailable β€” could not parse LLM response.",
extracted_variables={},
token_usage=token_usage,
)
parsed_response["step"] = step
parsed_response["token_usage"] = token_usage
# ── LOG ANALYSIS RESULTS ───────────────────────────────────────────────
logger.info("[ANALYZER] βœ” Analysis complete for Step %s:", step)
logger.info("[ANALYZER] Status : %s", parsed_response.get("status"))
logger.info("[ANALYZER] Extracted Variables : %s", json.dumps(parsed_response.get("extracted_variables", {})))
logger.info("[ANALYZER] OTP Required : %s", parsed_response.get("is_otp_required"))
logger.info("[ANALYZER] Selection Required : %s", parsed_response.get("is_selection_required"))
if parsed_response.get("is_selection_required"):
logger.info("[ANALYZER] Selection Field : %s", parsed_response.get("selection_field_name"))
opts = parsed_response.get("selection_options") or []
logger.info("[ANALYZER] Selection Options : %s", json.dumps(opts[:5])) # max 5 for brevity
logger.info("[ANALYZER] Token usage: %s", token_usage)
logger.info("="*70)
return AnalyzeResponse(**parsed_response)