ai-api-testing / backend /services /api_executor.py
Amitkasar's picture
solved issue
f1c3c6d
Raw History Blame Contribute Delete
9.76 kB
import json
from typing import Dict, Any, List
from langchain_core.prompts import PromptTemplate
from langchain_core.output_parsers import JsonOutputParser
from pydantic import BaseModel, Field
from backend.core.errors import AppError
from backend.core.logging import get_logger
from backend.integrations.llm.base import BaseLLMClient
logger = get_logger(__name__)
# Using Pydantic to structure LangChain output for step execution
class APIRequest(BaseModel):
method: str
url: str
headers: Dict[str, Any] = Field(default_factory=dict)
query_params: Dict[str, Any] = Field(default_factory=dict)
form_data: Dict[str, Any] = Field(default_factory=dict)
body: Any = Field(default_factory=dict)
files: Dict[str, str] = Field(default_factory=dict) # NEW: {fieldName: fileName}
class StepAnalysis(BaseModel):
required_inputs: List[Any] = Field(default_factory=list)
values_from_previous_steps: List[Any] = Field(default_factory=list)
generated_test_data: List[Any] = Field(default_factory=list)
class StepExecutionPlan(BaseModel):
step: int
request: APIRequest
analysis: StepAnalysis
# Prompt template for step preparation
STEP_PROMPT = """
You are an API execution assistant. Your job is to build a precise HTTP request for the CURRENT step.
NOTATION IN SPEC:
- {{prop*(str)}}: 'prop' is a REQUIRED string.
- {{prop(int)}}: 'prop' is an OPTIONAL integer.
- {{status(str[active|inactive])}}: 'status' is a string that MUST be one of the enum values [active, inactive].
- {{date(string:date-time)}}: 'date' is a string in date-time format.
INPUTS:
1. API Spec (current endpoint): {endpoint_spec}
2. Base URL: {base_url}
3. Scenario: {scenario}
4. Current Step: {step_definition}
5. User Credentials: {user_credentials}
6. Context Variables (Priority): {context_variables}
7. Assets (Semantic Context): {available_assets}
CORE EXECUTION RULES:
1. STRICT SCHEMA ADHERENCE (CRITICAL):
- You MUST ONLY use parameters, headers, and body fields that are explicitly defined in the API Spec. DO NOT hallucinate variables or fields that do not exist.
- Respect data types. If an integer is expected, output an integer, not a string.
2. URL & PATH PARAMETERS:
- You MUST replace all path variables like {{id}} or {{bus_id}} with actual values from Context or Step Definition.
- Example: If URL is "/buses/{{bus_id}}" and Context has bus_id=101, result URL must be "/buses/101".
3. AUTHORIZATION:
- Use credentials from "User Credentials" or "Context Variables" to satisfy authentication (token, api_key, etc).
- Only include an "Authorization" header if required by the spec. Use the correct format (e.g., "Bearer <token>", "Token <token>", or "Basic <base64>") as needed.
- DO NOT default to "Bearer" if the credential type is not specified.
4. DATA SOURCE PRIORITY:
- Priority 1: Context Variables (from previous steps).
- Priority 2: User Credentials (for login/auth).
- Priority 3: Step Definition / Scenario data.
5. STRICT ENUM ADHERENCE:
- If a field has an enum like [A|B|C], you MUST pick one of those exact values.
6. FORM vs JSON:
- Content-Type: application/x-www-form-urlencoded -> use 'form_data'.
- Content-Type: application/json -> use 'body'.
- If the endpoint receives a JSON array, the 'body' field must be an array instead of a dict.
7. FILES: Use 'files' object {{"field": "filename"}} using names from the provided Assets.
8. NO TEMPLATE STRINGS:
- Do not leave {{var}} or <var> syntax in the final 'request' JSON. Fully resolve variables.
9. COOKIE RULE:
- Do NOT add a Cookie header unless the spec explicitly requires it. Bearer token auth does not use cookies.
10. PHP SINGLE-FILE API RULE:
- When the Base URL path ends with a .php filename (e.g. mobile_api.php), the `action` routing parameter MUST be in `query_params` (e.g. {{"action": "login", "v": 1}}).
- For POST/PUT/PATCH requests you may also include `action` in the JSON body, but it is NOT needed for GET requests.
- NEVER append the semantic OpenAPI path (e.g. /auth/login) after the .php filename β€” the URL must stay exactly at the .php file and route via ?action= only.
- Correct URL for login: "https://host/mobile_api.php", query_params: {{"action": "login", "v": 1}}
11. MISSING ID RULE (CRITICAL):
- If a required path parameter (e.g. {{bus_id}}) or required body field is an ID that is NOT present in Context Variables and cannot be reasonably generated (it is a real DB record ID, not a UUID you create), do NOT guess a value.
- Instead, set the parameter to the literal string "__MISSING__<param_name>__" (e.g. "__MISSING__bus_id__") so the caller can detect and block execution.
- Examples of IDs you MUST NOT guess: bus_id, product_id, order_id, seat_id, route_id, ticket_id.
- Examples of IDs you CAN generate: new UUIDs for user registration, random test email addresses.
RETURN JSON ONLY:
{{
"step": 1,
"request": {{
"method": "POST", "url": "<FULL_URL_WITH_PARAMS_REPLACED>", "headers": {{"Authorization": "..."}},
"query_params": {{}}, "form_data": {{}}, "body": {{}}, "files": {{}}
}},
"analysis": {{
"required_inputs": ["list_of_fields_you_filled"],
"values_from_previous_steps": ["list_of_fields_taken_from_context"],
"generated_test_data": ["list_of_fields_you_invented"]
}}
}}
"""
def prepare_step_request(
endpoint_spec: Dict[str, Any],
base_url: str,
scenario: str,
step_definition: Dict[str, Any],
context_variables: Dict[str, Any],
user_credentials: Dict[str, Any] = None,
available_assets: Dict[str, Any] = None,
llm: BaseLLMClient = None,
) -> StepExecutionPlan:
if user_credentials is None:
user_credentials = {}
if available_assets is None:
available_assets = {}
parser = JsonOutputParser(pydantic_object=StepExecutionPlan)
prompt = PromptTemplate(
template=STEP_PROMPT,
input_variables=["endpoint_spec", "base_url", "scenario", "step_definition", "user_credentials", "context_variables", "available_assets"],
)
# ── PHASE 1: LOG EXECUTOR INPUTS ─────────────────────────────────────────────
step_num = step_definition.get("step", "?")
endpoint_label = step_definition.get("endpoint", "?")
logger.info("="*70)
logger.info("[EXECUTOR] β–Ά STEP %s PREPARATION STARTED: %s", step_num, endpoint_label)
logger.info("[EXECUTOR] Base URL : %s", base_url)
logger.info("[EXECUTOR] Step Definition : %s", json.dumps(step_definition))
logger.info("[EXECUTOR] Credentials : %s", json.dumps(user_credentials))
logger.info("[EXECUTOR] Context Variables : %s", json.dumps(context_variables))
logger.info("[EXECUTOR] Endpoint Spec : %s", json.dumps(endpoint_spec) if isinstance(endpoint_spec, dict) else str(endpoint_spec))
logger.info("[EXECUTOR] Available Assets : %s", json.dumps(list(available_assets.keys()) if isinstance(available_assets, dict) else []))
logger.info("="*70)
msg = prompt.format_prompt(
endpoint_spec=json.dumps(endpoint_spec, separators=(',', ':')),
base_url=base_url,
scenario=scenario,
step_definition=json.dumps(step_definition, separators=(',', ':')),
user_credentials=json.dumps(user_credentials, separators=(',', ':')),
context_variables=json.dumps(context_variables, separators=(',', ':')),
available_assets=json.dumps(available_assets, separators=(',', ':'))
).to_messages()
# ── PHASE 2: LLM CALL ────────────────────────────────────────────────────
logger.info("[EXECUTOR] ↑ Sending prompt to LLM...")
content, token_usage = llm.invoke(msg)
logger.info("[EXECUTOR] ↓ Raw LLM response:")
logger.info("-"*70)
logger.info("%s", content)
logger.info("-"*70)
# ── PHASE 3: PARSE ───────────────────────────────────────────────────────
try:
parsed_response = parser.parse(content)
except Exception as e:
logger.warning("[EXECUTOR] ⚠ Step request parse failed, retrying: %s", e)
content, token_usage = llm.invoke(msg)
logger.info("[EXECUTOR] ↓ Raw LLM response (retry):")
logger.info("-"*70)
logger.info("%s", content)
logger.info("-"*70)
try:
parsed_response = parser.parse(content)
except Exception as e2:
raise AppError(500, f"LLM returned unparseable output after 2 attempts: {e2}")
# ── PHASE 4: LOG FINAL REQUEST ──────────────────────────────────────────────
req = parsed_response.get("request", {})
logger.info("[EXECUTOR] βœ” Request plan generated for step %s:", step_num)
logger.info("[EXECUTOR] Method : %s", req.get("method"))
logger.info("[EXECUTOR] URL : %s", req.get("url"))
logger.info("[EXECUTOR] Query Params: %s", json.dumps(req.get("query_params", {})))
logger.info("[EXECUTOR] Headers : %s", json.dumps(req.get("headers", {})))
logger.info("[EXECUTOR] Body : %s", json.dumps(req.get("body", {})))
logger.info("[EXECUTOR] Form Data : %s", json.dumps(req.get("form_data", {})))
logger.info("[EXECUTOR] Files : %s", json.dumps(req.get("files", {})))
logger.info("[EXECUTOR] Token usage : %s", token_usage)
logger.info("="*70)
return StepExecutionPlan(**parsed_response), token_usage