import json from typing import Any, Dict, List, Optional from langchain_core.prompts import PromptTemplate from langchain_core.output_parsers import JsonOutputParser from backend.core.logging import get_logger from backend.integrations.llm.base import BaseLLMClient from backend.schemas import ExecutionPlan logger = get_logger(__name__) # Prompt template from the user prompt PLANNER_PROMPT = """ You are an intelligent AI API testing planner and execution assistant. Your job is to read API documentation extracted from a Swagger/OpenAPI file and execute API testing scenarios provided by the user in natural language. You must perform reasoning before every step and plan API execution carefully. INPUTS PROVIDED TO YOU 1. Parsed Swagger/OpenAPI documentation containing endpoints, methods, parameters, etc. 2. Base URL of the API server. 3. A tester scenario written in natural language. OBJECTIVE You must convert the tester scenario into a sequence of API calls and manage dependencies between requests. EXECUTION PLAN FORMAT You must return ONLY a JSON object representing the execution plan. DO NOT include markdown formatting or explanations. The format must exactly match: {{ "execution_plan": [ {{ "step": 1, "endpoint": "GET /buses", "purpose": "Fetch available buses so user can select a real bus_id", "required_context_keys": [] }}, {{ "step": 2, "endpoint": "POST /buses/{{bus_id}}/book", "purpose": "Book the selected bus", "required_context_keys": ["bus_id"] }} ], "suggested_credentials": {{ "username": "tester_user", "password": "Password123!", "email": "tester@example.com" }} }} DEPENDENCY RULE (CRITICAL): - Inspect every endpoint in your plan for path parameters (e.g. {{bus_id}}, {{product_id}}, {{seat_id}}). - If a path parameter cannot be known in advance (not in USER CREDENTIALS, not a UUID to generate, not derivable from a prior step in THIS plan), you MUST insert a GET/list step immediately before the step that needs it. That GET step should fetch the list of available options. - Example: "POST /buses/{{bus_id}}/book" at step 1 → insert "GET /buses" at step 1, shift booking to step 2. - Example: "GET /routes/{{route_id}}/stops" where route_id comes from a previous GET /routes → no insertion needed. REQUIRED_CONTEXT_KEYS RULE: - For each step, set "required_context_keys" to a list of context variable names that MUST be present before this step can run and that the AI cannot safely generate (real IDs, external references). - If the step needs no special context, set "required_context_keys" to []. - Example: step "POST /bookings/{{bus_id}}" → required_context_keys: ["bus_id"] - Example: step "POST /auth/login" with email/password from credentials → required_context_keys: [] API DOCUMENTATION: {parsed_spec} BASE URL: {base_url} TESTER SCENARIO: {scenario} USER CREDENTIALS (If present, user already exists): {user_credentials_str} IMPORTANT: - If the USER CREDENTIALS section above contains data (i.e. is not an empty {{}} object), it means a test user profile already exists. You MUST NOT include any steps in your execution plan to register, sign up, or create a new user. You should skip user creation entirely and proceed directly with login or other actions. - If the USER CREDENTIALS section is empty AND your plan requires authentication or user creation, you MUST provide a "suggested_credentials" object with realistic data (username, password, name, etc.) that can be used for these new accounts. """ def _condense_spec_for_planning(parsed_spec: Dict[str, Any]) -> List[Dict[str, Any]]: """Return a minimal endpoint list (method, path, summary) for the planning prompt. Full schemas are NOT included — they're fetched per-step during execution. This keeps the planning prompt small even for 100+ endpoint specs.""" endpoints = [] # OpenAPI / Swagger format paths = parsed_spec.get("paths", {}) for path, methods in paths.items(): if not isinstance(methods, dict): continue for method, op in methods.items(): if method.lower() in {"get", "post", "put", "patch", "delete", "head", "options"}: endpoints.append({ "method": method.upper(), "path": path, "summary": op.get("summary", op.get("description", ""))[:120] if isinstance(op, dict) else "", "tags": op.get("tags", [])[:3] if isinstance(op, dict) else [], }) # Postman / custom pre-parsed list if not endpoints and "endpoints" in parsed_spec: raw = parsed_spec["endpoints"] if isinstance(raw, list): for ep in raw: if isinstance(ep, dict): endpoints.append({ "method": ep.get("method", "GET"), "path": ep.get("path", ep.get("url", "")), "summary": ep.get("name", ep.get("summary", ""))[:120], }) else: # dict keyed by path — same as paths above return _condense_spec_for_planning({"paths": raw}) return endpoints def generate_execution_plan(parsed_spec: Dict[str, Any], base_url: str, scenario: str, user_credentials: Optional[Dict[str, Any]] = None, llm: BaseLLMClient = None) -> ExecutionPlan: """ Uses Mistral AI via LangChain to generate an execution plan based on the scenario. """ parser = JsonOutputParser(pydantic_object=ExecutionPlan) prompt = PromptTemplate( template=PLANNER_PROMPT, input_variables=["parsed_spec", "base_url", "scenario", "user_credentials_str"], ) condensed = _condense_spec_for_planning(parsed_spec) spec_payload = condensed if condensed else parsed_spec.get("endpoints", parsed_spec) # ── PHASE 1: LOG PLANNING INPUTS ───────────────────────────────────────── logger.info("="*70) logger.info("[PLANNER] ▶ PLAN GENERATION STARTED") logger.info("[PLANNER] Base URL : %s", base_url) logger.info("[PLANNER] Scenario : %s", scenario) logger.info("[PLANNER] Credentials : %s", json.dumps(user_credentials or {})) logger.info("[PLANNER] Condensed Spec (%d endpoints):", len(condensed)) for ep in condensed: logger.info("[PLANNER] %s %s — %s", ep.get("method",""), ep.get("path",""), ep.get("summary","")[:80]) logger.info("="*70) msg = prompt.format_prompt( parsed_spec=json.dumps(spec_payload, separators=(',', ':')), base_url=base_url, scenario=scenario, user_credentials_str=json.dumps(user_credentials or {}, separators=(',', ':')) ).to_messages() # ── PHASE 2: LLM CALL ──────────────────────────────────────────────────── logger.info("[PLANNER] ↑ Sending prompt to LLM...") content, token_usage = llm.invoke(msg) logger.info("[PLANNER] ↓ Raw LLM response:") logger.info("-"*70) logger.info("%s", content) logger.info("-"*70) # ── PHASE 3: PARSE ─────────────────────────────────────────────────────── try: parsed_response = parser.parse(content) except Exception as parse_err: logger.warning("[PLANNER] ⚠ Plan parse failed, retrying: %s", parse_err) content, token_usage = llm.invoke(msg) logger.info("[PLANNER] ↓ Raw LLM response (retry):") logger.info("-"*70) logger.info("%s", content) logger.info("-"*70) parsed_response = parser.parse(content) # ── PHASE 4: LOG FINAL PLAN ────────────────────────────────────────────── logger.info("[PLANNER] ✔ Execution plan generated — %d step(s):", len(parsed_response.get("execution_plan", []))) for step in parsed_response.get("execution_plan", []): logger.info( "[PLANNER] Step %s: %s | purpose: %s | required_context: %s", step.get("step"), step.get("endpoint"), step.get("purpose", "")[:80], step.get("required_context_keys", []) ) if parsed_response.get("suggested_credentials"): logger.info("[PLANNER] Suggested credentials: %s", json.dumps(parsed_response["suggested_credentials"])) logger.info("[PLANNER] Token usage: %s", token_usage) logger.info("="*70) # Return as Pydantic model return ExecutionPlan( execution_plan=parsed_response["execution_plan"], suggested_credentials=parsed_response.get("suggested_credentials"), token_usage=token_usage )