ai-api-testing / backend /app /api /routes /execution.py
Amitkasar's picture
solved issue
f1c3c6d
Raw History Blame Contribute Delete
25.2 kB
import asyncio
import json
import os
import re
import traceback
from urllib.parse import parse_qsl, urlencode, urlparse
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
from backend import database, schemas
from backend.core.crypto import decrypt_credentials, encrypt_credentials
from backend.core.limiter import limiter
from backend.core.logging import get_logger
from backend.core.url_guard import resolve_safe_ip
from backend.services.api_executor import prepare_step_request
from backend.services.asset_service import AssetService, storage
from backend.services.context_manager import (
set_context_variables,
)
from backend.integrations.llm.resolver import get_llm_for_task
from backend.services.llm_planner import generate_execution_plan
from backend.services.response_analyzer import analyze_api_response
from backend.services.usage_service import check_budget, log_usage
router = APIRouter()
logger = get_logger(__name__)
PLACEHOLDER_HOSTS = {"api.example.com", "example.com", "<base_url>", "your-api.com"}
def _resolve_refs(obj, full_spec, _resolving=None):
if _resolving is None:
_resolving = set()
if isinstance(obj, dict):
if "$ref" in obj:
ref = obj["$ref"]
if ref in _resolving:
return {"$ref_loop": ref}
_resolving = _resolving | {ref}
parts = ref.lstrip("#/").split("/")
target = full_spec
for part in parts:
target = target.get(part, {}) if isinstance(target, dict) else {}
return _resolve_refs(target, full_spec, _resolving)
return {k: _resolve_refs(v, full_spec, _resolving) for k, v in obj.items()}
elif isinstance(obj, list):
return [_resolve_refs(item, full_spec, _resolving) for item in obj]
return obj
def _shorthand_spec(obj):
if isinstance(obj, dict):
if "type" in obj:
t = obj["type"]
fmt = f":{obj['format']}" if "format" in obj else ""
enum_vals = f"[{'|'.join(map(str, obj['enum']))}]" if "enum" in obj else ""
if t == "object" and "properties" in obj:
req_list = obj.get("required", [])
props = [
f"{k}{'*' if k in req_list or v.get('nullable') == False else ''}({_shorthand_spec(v)})"
for k, v in obj["properties"].items()
]
return "{" + ",".join(props) + "}"
if t == "array" and "items" in obj:
return f"[{_shorthand_spec(obj['items'])}]"
return f"{t}{fmt}{enum_vals}"
return {k: _shorthand_spec(v) for k, v in obj.items()}
return str(obj)
def _prune_spec(obj):
DROP_KEYS = {"description", "summary", "example", "examples", "format", "title", "default", "enum_descriptions"}
if isinstance(obj, dict):
return {k: _prune_spec(v) for k, v in obj.items() if k not in DROP_KEYS}
elif isinstance(obj, list):
return [_prune_spec(item) for item in obj]
return obj
def _match_path(path_template: str, actual_path: str) -> bool:
pattern = re.sub(r'\{[^}]+\}', r'[^/]+', path_template)
pattern = "^" + pattern.rstrip("/") + "/?$"
return bool(re.match(pattern, actual_path.split("?")[0]))
def _is_placeholder(url: str) -> bool:
try:
host = urlparse(url).hostname or ""
return host.lower() in PLACEHOLDER_HOSTS or url.startswith("<BASE_URL>")
except Exception:
return False
def _strip_phantom_path(url: str, base_url: str) -> str:
"""Remove path segments the LLM appended beyond a file-based base URL.
Single-file APIs (e.g. admin_api.php) route via ?action=, not URL paths.
When the OpenAPI spec defines /auth/me and the LLM appends that to
admin_api.php, the result is admin_api.php/auth/me which 404s.
We detect this by checking whether the base URL's last path segment has
a file extension — if so, strip any extra segments the LLM added.
"""
try:
base_parsed = urlparse(base_url if "://" in base_url else "https://" + base_url)
base_file = base_parsed.path.rstrip("/").split("/")[-1]
# Only act when base URL ends with a file (has an extension)
if "." not in base_file:
return url
base_path = base_parsed.path.rstrip("/")
url_parsed = urlparse(url)
if url_parsed.path.startswith(base_path) and url_parsed.path != base_path:
from urllib.parse import urlunparse
# Preserve the query string — the caller's merged_query will merge it
# with any explicit query_params from the LLM, so action= etc. survive.
fixed = urlunparse(url_parsed._replace(path=base_path))
logger.info("Stripped phantom path '%s' → '%s'", url, fixed)
return fixed
except Exception:
pass
return url
@router.post("/plan", response_model=schemas.ExecutionPlan)
@limiter.limit("10/minute")
async def generate_plan(request: Request, body: schemas.PlanRequest, db: Session = Depends(database.get_db)):
try:
check_budget(db, body.session_id)
parsed_spec = json.loads(body.spec_content) if isinstance(body.spec_content, str) else body.spec_content
credentials = body.user_credentials or {}
if body.spec_id:
spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first()
if spec and spec.login_credentials:
credentials = decrypt_credentials(spec.login_credentials)
llm = get_llm_for_task("plan_generation", db)
plan = await asyncio.to_thread(generate_execution_plan, parsed_spec, body.base_url, body.scenario, credentials, llm)
log_usage(db, body.session_id, None, "plan", plan.token_usage.model_dump() if plan.token_usage else None)
if body.spec_id:
spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first()
if spec:
spec.execution_plan = plan.model_dump()["execution_plan"]
spec.scenario = body.scenario
flag_modified(spec, "execution_plan")
flag_modified(spec, "scenario")
if body.session_id:
session = db.query(database.SessionModel).filter(
database.SessionModel.id == body.session_id
).first()
if session:
session.spec_id = spec.id
session.spec_name = spec.name
session.scenario = body.scenario
session.base_url = body.base_url
if plan.suggested_credentials and isinstance(plan.suggested_credentials, dict):
current_creds = decrypt_credentials(dict(spec.login_credentials or {}))
if any(k not in current_creds or current_creds[k] != v for k, v in plan.suggested_credentials.items()):
current_creds.update(plan.suggested_credentials)
spec.login_credentials = encrypt_credentials(current_creds)
flag_modified(spec, "login_credentials")
logger.info("AI suggested credentials merged for spec '%s'", spec.name)
db.commit()
return plan
except Exception as e:
raise HTTPException(status_code=500, detail=f"Planning failed: {str(e)}")
def _sse(payload: dict) -> str:
return f"data: {json.dumps(payload)}\n\n"
@router.post("/plan/stream")
@limiter.limit("10/minute")
async def stream_plan(request: Request, body: schemas.PlanRequest, db: Session = Depends(database.get_db)):
"""Server-Sent Events endpoint for /plan — streams steps as they become available so the
frontend can render progress instead of waiting for the full response."""
async def generate():
try:
check_budget(db, body.session_id)
yield _sse({"type": "status", "message": "Analysing spec and scenario…"})
parsed_spec = json.loads(body.spec_content) if isinstance(body.spec_content, str) else body.spec_content
credentials = body.user_credentials or {}
if body.spec_id:
spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first()
if spec and spec.login_credentials:
credentials = decrypt_credentials(spec.login_credentials)
yield _sse({"type": "status", "message": "Generating execution plan with AI…"})
llm = get_llm_for_task("plan_generation", db)
plan = await asyncio.to_thread(
generate_execution_plan, parsed_spec, body.base_url, body.scenario, credentials, llm
)
log_usage(db, body.session_id, None, "plan", plan.token_usage.model_dump() if plan.token_usage else None)
if body.spec_id:
spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first()
if spec:
spec.execution_plan = plan.model_dump()["execution_plan"]
spec.scenario = body.scenario
flag_modified(spec, "execution_plan")
flag_modified(spec, "scenario")
if body.session_id:
session = db.query(database.SessionModel).filter(
database.SessionModel.id == body.session_id
).first()
if session:
session.spec_id = spec.id
session.spec_name = spec.name
session.scenario = body.scenario
session.base_url = body.base_url
if plan.suggested_credentials and isinstance(plan.suggested_credentials, dict):
current_creds = decrypt_credentials(dict(spec.login_credentials or {}))
if any(k not in current_creds or current_creds[k] != v for k, v in plan.suggested_credentials.items()):
current_creds.update(plan.suggested_credentials)
spec.login_credentials = encrypt_credentials(current_creds)
flag_modified(spec, "login_credentials")
db.commit()
# Stream each step individually so the UI can render them as they arrive
for step in plan.execution_plan:
yield _sse({"type": "step", "step": step.model_dump()})
await asyncio.sleep(0.04)
yield _sse({
"type": "complete",
"suggested_credentials": plan.suggested_credentials,
"token_usage": plan.token_usage.model_dump() if plan.token_usage else None,
})
except Exception as e:
logger.error("SSE plan stream error: %s", traceback.format_exc())
yield _sse({"type": "error", "message": str(e)})
return StreamingResponse(
generate(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
},
)
@router.post("/execute-step")
@limiter.limit("30/minute")
async def execute_step(request: Request, body: schemas.ExecuteStepRequest, db: Session = Depends(database.get_db)):
try:
check_budget(db, body.session_id)
endpoint_str = body.step_definition.get("endpoint", "")
actual_endpoint_spec = {"endpoint": endpoint_str}
if body.spec_id:
spec_doc = db.query(database.SpecDocument).filter(
database.SpecDocument.id == body.spec_id
).first()
if spec_doc and spec_doc.parsed_json:
full_spec = spec_doc.parsed_json
paths = full_spec.get("paths", {})
parts = endpoint_str.split(" ", 1)
path_only = parts[1] if len(parts) == 2 else parts[0]
matched_path = path_only if path_only in paths else next(
(p for p in paths if _match_path(p, path_only)), None
)
if matched_path:
raw_spec = {matched_path: paths[matched_path]}
resolved = _resolve_refs(raw_spec, full_spec)
path_data = resolved.get(matched_path, {})
for method in path_data:
if isinstance(path_data[method], dict):
path_data[method].pop("responses", None)
actual_endpoint_spec = _shorthand_spec(_prune_spec(resolved))
else:
logger.warning("Path match failed for %s, falling back", path_only)
actual_endpoint_spec = {"endpoint": endpoint_str, "warning": "Full schema match failed"}
rich_assets = AssetService.get_ai_context(db, body.spec_id)
effective_credentials = body.user_credentials
if not effective_credentials and body.spec_id:
spec = db.query(database.SpecDocument).filter(
database.SpecDocument.id == body.spec_id
).first()
if spec and spec.login_credentials:
effective_credentials = decrypt_credentials(spec.login_credentials)
logger.info("Using stored credentials for spec '%s'", spec.name)
execution_context = dict(body.context_variables or {})
step_llm = get_llm_for_task("step_execution", db)
req_plan, executor_token_usage = await asyncio.to_thread(
prepare_step_request,
endpoint_spec=actual_endpoint_spec,
base_url=body.base_url,
scenario="",
step_definition=body.step_definition,
context_variables=execution_context,
user_credentials=effective_credentials,
available_assets=rich_assets,
llm=step_llm,
)
# Guard: detect sentinel values that mean the AI couldn't resolve a real ID
_MISSING_PATTERN = re.compile(r"__MISSING__(\w+)__")
missing_in_url = _MISSING_PATTERN.findall(req_plan.request.url)
missing_in_body = _MISSING_PATTERN.findall(json.dumps(req_plan.request.body or {}))
all_missing = list(set(missing_in_url + missing_in_body))
if all_missing:
raise HTTPException(
status_code=400,
detail={
"error": "missing_context_keys",
"message": f"Step requires real IDs that are not in context: {all_missing}. Please provide them manually before executing this step.",
"missing_keys": all_missing,
},
)
url = req_plan.request.url.strip()
if url.startswith("/"):
url = body.base_url.rstrip("/") + url
logger.info("[ROUTE] Relative URL resolved to: %s", url)
elif _is_placeholder(url):
parsed = urlparse(url)
path = parsed.path or "/"
url = body.base_url.rstrip("/") + path
if parsed.query:
url += "?" + parsed.query
logger.info("[ROUTE] Placeholder URL replaced : %s", url)
elif not url.startswith("http"):
url = f"https://{url}"
logger.info("[ROUTE] Scheme-prefixed URL : %s", url)
url = _strip_phantom_path(url, body.base_url)
# Merge query params from the URL and from the LLM's query_params dict into
# a single authoritative dict, then strip the query string from the URL.
parsed = urlparse(url)
merged_query = dict(parse_qsl(parsed.query))
merged_query.update(req_plan.request.query_params or {})
url_no_query = url.split("?")[0]
logger.info("[ROUTE] URL (no query) : %s", url_no_query)
logger.info("[ROUTE] Merged query : %s", merged_query)
# PHP single-file APIs that read the action from php://input instead of $_GET:
# inject action into the JSON body for mutating methods only. GET requests
# pass action via query string ($_GET) — adding a body to GET is non-standard
# and may confuse the server or proxies.
_method_upper = req_plan.request.method.strip().upper()
if url_no_query.endswith(".php") and "action" in merged_query and _method_upper != "GET":
_action = merged_query["action"]
if isinstance(req_plan.request.body, dict):
req_plan.request.body.setdefault("action", _action)
elif req_plan.request.body is None:
req_plan.request.body = {"action": _action}
# Resolve IP once to block private/internal addresses (SSRF guard).
# We use the original hostname URL for the actual request so TLS certificate
# verification works correctly against the hostname (not the raw IP).
resolve_safe_ip(url)
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
req_headers = {k: v for k, v in req_plan.request.headers.items() if k.lower() != "cookie"}
# Browser User-Agent to pass WAF/Cloudflare checks.
if not any(k.lower() == "user-agent" for k in req_headers):
req_headers["User-Agent"] = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
if not any(k.lower() == "accept" for k in req_headers):
req_headers["Accept"] = "application/json"
if not any(k.lower() == "accept-language" for k in req_headers):
req_headers["Accept-Language"] = "en-US,en;q=0.9"
# Embed query params directly into the URL string — same approach as
# the Node.js test script which hardcodes ?action=me in the URL literal.
final_url = (
f"{url_no_query}?{urlencode(merged_query)}" if merged_query else url_no_query
)
http_req = client.build_request(
method=req_plan.request.method.strip().upper(),
url=final_url,
headers=req_headers,
data=req_plan.request.form_data if req_plan.request.form_data else None,
json=req_plan.request.body if req_plan.request.body and not req_plan.request.form_data else None,
)
# Log the ACTUAL URL and headers httpx will send — check backend console
logger.info("="*70)
logger.info("[ROUTE] >>> OUTGOING %s %s", http_req.method, http_req.url)
logger.info("[ROUTE] >>> HEADERS %s", dict(http_req.headers))
if req_plan.request.body:
logger.info("[ROUTE] >>> BODY %s", json.dumps(req_plan.request.body))
if req_plan.request.form_data:
logger.info("[ROUTE] >>> FORMDATA %s", json.dumps(req_plan.request.form_data))
logger.info("="*70)
files_payload = {}
opened_files = []
try:
if req_plan.request.files:
req_headers = {
k: v for k, v in req_headers.items() if k.lower() != "content-type"
}
for field_name, file_name in req_plan.request.files.items():
asset = db.query(database.Asset).filter(database.Asset.name == file_name).first()
if asset:
full_path = storage.get_full_path(asset.file_path)
if os.path.exists(full_path):
f = open(full_path, "rb")
opened_files.append(f)
files_payload[field_name] = (asset.name, f, asset.file_type)
else:
logger.warning("Asset file not found on disk: %s", full_path)
else:
logger.warning("Asset not found in Media Hub: %s", file_name)
logger.info("Executing %s %s", http_req.method, http_req.url)
try:
if files_payload:
response = await client.request(
method=req_plan.request.method.strip().upper(),
url=final_url,
headers=req_headers,
data=req_plan.request.form_data if req_plan.request.form_data else None,
files=files_payload,
)
else:
response = await client.send(http_req)
except httpx.TimeoutException:
raise HTTPException(
status_code=504,
detail=f"Request to {url} timed out after {120}s. Check the API server is reachable.",
)
except httpx.ConnectError as e:
raise HTTPException(
status_code=502,
detail=f"Could not connect to {url}: {str(e)}",
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Request to {url} failed: {str(e)}")
finally:
for f in opened_files:
f.close()
body_text = response.text
resp_data = {
"status_code": response.status_code,
"body": body_text,
"headers": dict(response.headers),
}
resp_data["cookies"] = {}
# Log incoming response — always visible in terminal
logger.info("="*70)
logger.info("[ROUTE] <<< INCOMING STATUS %s from %s", response.status_code, url_no_query)
logger.info("[ROUTE] <<< RESPONSE HEADERS %s", dict(response.headers))
logger.info("[ROUTE] <<< RESPONSE BODY (first 1000 chars):\n%s", body_text[:1000])
logger.info("="*70)
# Log a clear error line for 4xx/5xx so it's visible in the backend console
if response.status_code >= 400:
snippet = body_text[:300].replace("\n", " ")
logger.warning(
"[ROUTE] Step %s returned %s from %s: %s",
req_plan.step, response.status_code, url, snippet,
)
req_plan.request.url = url
db.add(database.ExecutionHistory(
session_id=body.session_id,
step=req_plan.step,
request_json=req_plan.request.model_dump(),
response_json=resp_data,
step_definition_json=body.step_definition,
))
db.commit()
log_usage(db, body.session_id, None, "execute_step", executor_token_usage)
return {
"executed_request": req_plan.request.model_dump(),
"response": resp_data,
"analysis_needed": req_plan.step,
"token_usage": executor_token_usage,
}
except Exception as e:
logger.error("Execution error: %s", traceback.format_exc())
raise HTTPException(status_code=500, detail=f"Execution failed: {str(e)}")
@router.post("/analyze-response", response_model=schemas.AnalyzeResponse)
async def analyze_response_endpoint(request: schemas.AnalyzeRequest, db: Session = Depends(database.get_db)):
try:
step_def = request.step_definition or {}
analysis_llm = get_llm_for_task("response_analysis", db)
analysis = await asyncio.to_thread(
analyze_api_response,
step=request.step,
endpoint=step_def.get("endpoint", "Unknown"),
purpose=step_def.get("purpose", "Unknown"),
status_code=request.response_data.get("status_code", 0),
response_body=request.response_data.get("body", ""),
llm=analysis_llm,
)
log_usage(db, request.session_id, None, "analyze", analysis.token_usage.model_dump() if analysis.token_usage else None)
if analysis.extracted_variables:
set_context_variables(db, request.session_id, analysis.extracted_variables)
history = (
db.query(database.ExecutionHistory)
.filter(
database.ExecutionHistory.session_id == request.session_id,
database.ExecutionHistory.step == request.step,
)
.order_by(database.ExecutionHistory.id.desc())
.first()
)
if history:
history.analysis_json = analysis.model_dump()
db.commit()
return analysis
except Exception as e:
raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")