Spaces:
Runtime error
Runtime error
Download backend/services/llm_planner.py from Amitkasar/ai-api-testing: direct link, hf CLI and curl.
- Browser
- Download file 8.95 kB
-
https://huggingface.co/spaces/Amitkasar/ai-api-testing/resolve/main/backend/services/llm_planner.py
- Command line
-
hf download hf://spaces/Amitkasar/ai-api-testing/backend/services/llm_planner.py
-
curl -L -o llm_planner.py https://huggingface.co/spaces/Amitkasar/ai-api-testing/resolve/main/backend/services/llm_planner.py
8.95 kB
| import json | |
| from typing import Any, Dict, List, Optional | |
| from langchain_core.prompts import PromptTemplate | |
| from langchain_core.output_parsers import JsonOutputParser | |
| from backend.core.logging import get_logger | |
| from backend.integrations.llm.base import BaseLLMClient | |
| from backend.schemas import ExecutionPlan | |
| logger = get_logger(__name__) | |
| # Prompt template from the user prompt | |
| PLANNER_PROMPT = """ | |
| You are an intelligent AI API testing planner and execution assistant. | |
| Your job is to read API documentation extracted from a Swagger/OpenAPI file and execute API testing scenarios provided by the user in natural language. | |
| You must perform reasoning before every step and plan API execution carefully. | |
| INPUTS PROVIDED TO YOU | |
| 1. Parsed Swagger/OpenAPI documentation containing endpoints, methods, parameters, etc. | |
| 2. Base URL of the API server. | |
| 3. A tester scenario written in natural language. | |
| OBJECTIVE | |
| You must convert the tester scenario into a sequence of API calls and manage dependencies between requests. | |
| EXECUTION PLAN FORMAT | |
| You must return ONLY a JSON object representing the execution plan. DO NOT include markdown formatting or explanations. The format must exactly match: | |
| {{ | |
| "execution_plan": [ | |
| {{ | |
| "step": 1, | |
| "endpoint": "GET /buses", | |
| "purpose": "Fetch available buses so user can select a real bus_id", | |
| "required_context_keys": [] | |
| }}, | |
| {{ | |
| "step": 2, | |
| "endpoint": "POST /buses/{{bus_id}}/book", | |
| "purpose": "Book the selected bus", | |
| "required_context_keys": ["bus_id"] | |
| }} | |
| ], | |
| "suggested_credentials": {{ | |
| "username": "tester_user", | |
| "password": "Password123!", | |
| "email": "tester@example.com" | |
| }} | |
| }} | |
| DEPENDENCY RULE (CRITICAL): | |
| - Inspect every endpoint in your plan for path parameters (e.g. {{bus_id}}, {{product_id}}, {{seat_id}}). | |
| - If a path parameter cannot be known in advance (not in USER CREDENTIALS, not a UUID to generate, not derivable from a prior step in THIS plan), you MUST insert a GET/list step immediately before the step that needs it. That GET step should fetch the list of available options. | |
| - Example: "POST /buses/{{bus_id}}/book" at step 1 β insert "GET /buses" at step 1, shift booking to step 2. | |
| - Example: "GET /routes/{{route_id}}/stops" where route_id comes from a previous GET /routes β no insertion needed. | |
| REQUIRED_CONTEXT_KEYS RULE: | |
| - For each step, set "required_context_keys" to a list of context variable names that MUST be present before this step can run and that the AI cannot safely generate (real IDs, external references). | |
| - If the step needs no special context, set "required_context_keys" to []. | |
| - Example: step "POST /bookings/{{bus_id}}" β required_context_keys: ["bus_id"] | |
| - Example: step "POST /auth/login" with email/password from credentials β required_context_keys: [] | |
| API DOCUMENTATION: | |
| {parsed_spec} | |
| BASE URL: | |
| {base_url} | |
| TESTER SCENARIO: | |
| {scenario} | |
| USER CREDENTIALS (If present, user already exists): | |
| {user_credentials_str} | |
| IMPORTANT: | |
| - If the USER CREDENTIALS section above contains data (i.e. is not an empty {{}} object), it means a test user profile already exists. You MUST NOT include any steps in your execution plan to register, sign up, or create a new user. You should skip user creation entirely and proceed directly with login or other actions. | |
| - If the USER CREDENTIALS section is empty AND your plan requires authentication or user creation, you MUST provide a "suggested_credentials" object with realistic data (username, password, name, etc.) that can be used for these new accounts. | |
| """ | |
| def _condense_spec_for_planning(parsed_spec: Dict[str, Any]) -> List[Dict[str, Any]]: | |
| """Return a minimal endpoint list (method, path, summary) for the planning prompt. | |
| Full schemas are NOT included β they're fetched per-step during execution. | |
| This keeps the planning prompt small even for 100+ endpoint specs.""" | |
| endpoints = [] | |
| # OpenAPI / Swagger format | |
| paths = parsed_spec.get("paths", {}) | |
| for path, methods in paths.items(): | |
| if not isinstance(methods, dict): | |
| continue | |
| for method, op in methods.items(): | |
| if method.lower() in {"get", "post", "put", "patch", "delete", "head", "options"}: | |
| endpoints.append({ | |
| "method": method.upper(), | |
| "path": path, | |
| "summary": op.get("summary", op.get("description", ""))[:120] if isinstance(op, dict) else "", | |
| "tags": op.get("tags", [])[:3] if isinstance(op, dict) else [], | |
| }) | |
| # Postman / custom pre-parsed list | |
| if not endpoints and "endpoints" in parsed_spec: | |
| raw = parsed_spec["endpoints"] | |
| if isinstance(raw, list): | |
| for ep in raw: | |
| if isinstance(ep, dict): | |
| endpoints.append({ | |
| "method": ep.get("method", "GET"), | |
| "path": ep.get("path", ep.get("url", "")), | |
| "summary": ep.get("name", ep.get("summary", ""))[:120], | |
| }) | |
| else: | |
| # dict keyed by path β same as paths above | |
| return _condense_spec_for_planning({"paths": raw}) | |
| return endpoints | |
| def generate_execution_plan(parsed_spec: Dict[str, Any], base_url: str, scenario: str, user_credentials: Optional[Dict[str, Any]] = None, llm: BaseLLMClient = None) -> ExecutionPlan: | |
| """ | |
| Uses Mistral AI via LangChain to generate an execution plan based on the scenario. | |
| """ | |
| parser = JsonOutputParser(pydantic_object=ExecutionPlan) | |
| prompt = PromptTemplate( | |
| template=PLANNER_PROMPT, | |
| input_variables=["parsed_spec", "base_url", "scenario", "user_credentials_str"], | |
| ) | |
| condensed = _condense_spec_for_planning(parsed_spec) | |
| spec_payload = condensed if condensed else parsed_spec.get("endpoints", parsed_spec) | |
| # ββ PHASE 1: LOG PLANNING INPUTS βββββββββββββββββββββββββββββββββββββββββ | |
| logger.info("="*70) | |
| logger.info("[PLANNER] βΆ PLAN GENERATION STARTED") | |
| logger.info("[PLANNER] Base URL : %s", base_url) | |
| logger.info("[PLANNER] Scenario : %s", scenario) | |
| logger.info("[PLANNER] Credentials : %s", json.dumps(user_credentials or {})) | |
| logger.info("[PLANNER] Condensed Spec (%d endpoints):", len(condensed)) | |
| for ep in condensed: | |
| logger.info("[PLANNER] %s %s β %s", ep.get("method",""), ep.get("path",""), ep.get("summary","")[:80]) | |
| logger.info("="*70) | |
| msg = prompt.format_prompt( | |
| parsed_spec=json.dumps(spec_payload, separators=(',', ':')), | |
| base_url=base_url, | |
| scenario=scenario, | |
| user_credentials_str=json.dumps(user_credentials or {}, separators=(',', ':')) | |
| ).to_messages() | |
| # ββ PHASE 2: LLM CALL ββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| logger.info("[PLANNER] β Sending prompt to LLM...") | |
| content, token_usage = llm.invoke(msg) | |
| logger.info("[PLANNER] β Raw LLM response:") | |
| logger.info("-"*70) | |
| logger.info("%s", content) | |
| logger.info("-"*70) | |
| # ββ PHASE 3: PARSE βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| try: | |
| parsed_response = parser.parse(content) | |
| except Exception as parse_err: | |
| logger.warning("[PLANNER] β Plan parse failed, retrying: %s", parse_err) | |
| content, token_usage = llm.invoke(msg) | |
| logger.info("[PLANNER] β Raw LLM response (retry):") | |
| logger.info("-"*70) | |
| logger.info("%s", content) | |
| logger.info("-"*70) | |
| parsed_response = parser.parse(content) | |
| # ββ PHASE 4: LOG FINAL PLAN ββββββββββββββββββββββββββββββββββββββββββββββ | |
| logger.info("[PLANNER] β Execution plan generated β %d step(s):", len(parsed_response.get("execution_plan", []))) | |
| for step in parsed_response.get("execution_plan", []): | |
| logger.info( | |
| "[PLANNER] Step %s: %s | purpose: %s | required_context: %s", | |
| step.get("step"), step.get("endpoint"), step.get("purpose", "")[:80], | |
| step.get("required_context_keys", []) | |
| ) | |
| if parsed_response.get("suggested_credentials"): | |
| logger.info("[PLANNER] Suggested credentials: %s", json.dumps(parsed_response["suggested_credentials"])) | |
| logger.info("[PLANNER] Token usage: %s", token_usage) | |
| logger.info("="*70) | |
| # Return as Pydantic model | |
| return ExecutionPlan( | |
| execution_plan=parsed_response["execution_plan"], | |
| suggested_credentials=parsed_response.get("suggested_credentials"), | |
| token_usage=token_usage | |
| ) | |