import asyncio import json import os import re import traceback from urllib.parse import parse_qsl, urlencode, urlparse import httpx from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import StreamingResponse from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified from backend import database, schemas from backend.core.crypto import decrypt_credentials, encrypt_credentials from backend.core.limiter import limiter from backend.core.logging import get_logger from backend.core.url_guard import resolve_safe_ip from backend.services.api_executor import prepare_step_request from backend.services.asset_service import AssetService, storage from backend.services.context_manager import ( set_context_variables, ) from backend.integrations.llm.resolver import get_llm_for_task from backend.services.llm_planner import generate_execution_plan from backend.services.response_analyzer import analyze_api_response from backend.services.usage_service import check_budget, log_usage router = APIRouter() logger = get_logger(__name__) PLACEHOLDER_HOSTS = {"api.example.com", "example.com", "", "your-api.com"} def _resolve_refs(obj, full_spec, _resolving=None): if _resolving is None: _resolving = set() if isinstance(obj, dict): if "$ref" in obj: ref = obj["$ref"] if ref in _resolving: return {"$ref_loop": ref} _resolving = _resolving | {ref} parts = ref.lstrip("#/").split("/") target = full_spec for part in parts: target = target.get(part, {}) if isinstance(target, dict) else {} return _resolve_refs(target, full_spec, _resolving) return {k: _resolve_refs(v, full_spec, _resolving) for k, v in obj.items()} elif isinstance(obj, list): return [_resolve_refs(item, full_spec, _resolving) for item in obj] return obj def _shorthand_spec(obj): if isinstance(obj, dict): if "type" in obj: t = obj["type"] fmt = f":{obj['format']}" if "format" in obj else "" enum_vals = f"[{'|'.join(map(str, obj['enum']))}]" if "enum" in obj else "" if t == "object" and "properties" in obj: req_list = obj.get("required", []) props = [ f"{k}{'*' if k in req_list or v.get('nullable') == False else ''}({_shorthand_spec(v)})" for k, v in obj["properties"].items() ] return "{" + ",".join(props) + "}" if t == "array" and "items" in obj: return f"[{_shorthand_spec(obj['items'])}]" return f"{t}{fmt}{enum_vals}" return {k: _shorthand_spec(v) for k, v in obj.items()} return str(obj) def _prune_spec(obj): DROP_KEYS = {"description", "summary", "example", "examples", "format", "title", "default", "enum_descriptions"} if isinstance(obj, dict): return {k: _prune_spec(v) for k, v in obj.items() if k not in DROP_KEYS} elif isinstance(obj, list): return [_prune_spec(item) for item in obj] return obj def _match_path(path_template: str, actual_path: str) -> bool: pattern = re.sub(r'\{[^}]+\}', r'[^/]+', path_template) pattern = "^" + pattern.rstrip("/") + "/?$" return bool(re.match(pattern, actual_path.split("?")[0])) def _is_placeholder(url: str) -> bool: try: host = urlparse(url).hostname or "" return host.lower() in PLACEHOLDER_HOSTS or url.startswith("") except Exception: return False def _strip_phantom_path(url: str, base_url: str) -> str: """Remove path segments the LLM appended beyond a file-based base URL. Single-file APIs (e.g. admin_api.php) route via ?action=, not URL paths. When the OpenAPI spec defines /auth/me and the LLM appends that to admin_api.php, the result is admin_api.php/auth/me which 404s. We detect this by checking whether the base URL's last path segment has a file extension — if so, strip any extra segments the LLM added. """ try: base_parsed = urlparse(base_url if "://" in base_url else "https://" + base_url) base_file = base_parsed.path.rstrip("/").split("/")[-1] # Only act when base URL ends with a file (has an extension) if "." not in base_file: return url base_path = base_parsed.path.rstrip("/") url_parsed = urlparse(url) if url_parsed.path.startswith(base_path) and url_parsed.path != base_path: from urllib.parse import urlunparse # Preserve the query string — the caller's merged_query will merge it # with any explicit query_params from the LLM, so action= etc. survive. fixed = urlunparse(url_parsed._replace(path=base_path)) logger.info("Stripped phantom path '%s' → '%s'", url, fixed) return fixed except Exception: pass return url @router.post("/plan", response_model=schemas.ExecutionPlan) @limiter.limit("10/minute") async def generate_plan(request: Request, body: schemas.PlanRequest, db: Session = Depends(database.get_db)): try: check_budget(db, body.session_id) parsed_spec = json.loads(body.spec_content) if isinstance(body.spec_content, str) else body.spec_content credentials = body.user_credentials or {} if body.spec_id: spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() if spec and spec.login_credentials: credentials = decrypt_credentials(spec.login_credentials) llm = get_llm_for_task("plan_generation", db) plan = await asyncio.to_thread(generate_execution_plan, parsed_spec, body.base_url, body.scenario, credentials, llm) log_usage(db, body.session_id, None, "plan", plan.token_usage.model_dump() if plan.token_usage else None) if body.spec_id: spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() if spec: spec.execution_plan = plan.model_dump()["execution_plan"] spec.scenario = body.scenario flag_modified(spec, "execution_plan") flag_modified(spec, "scenario") if body.session_id: session = db.query(database.SessionModel).filter( database.SessionModel.id == body.session_id ).first() if session: session.spec_id = spec.id session.spec_name = spec.name session.scenario = body.scenario session.base_url = body.base_url if plan.suggested_credentials and isinstance(plan.suggested_credentials, dict): current_creds = decrypt_credentials(dict(spec.login_credentials or {})) if any(k not in current_creds or current_creds[k] != v for k, v in plan.suggested_credentials.items()): current_creds.update(plan.suggested_credentials) spec.login_credentials = encrypt_credentials(current_creds) flag_modified(spec, "login_credentials") logger.info("AI suggested credentials merged for spec '%s'", spec.name) db.commit() return plan except Exception as e: raise HTTPException(status_code=500, detail=f"Planning failed: {str(e)}") def _sse(payload: dict) -> str: return f"data: {json.dumps(payload)}\n\n" @router.post("/plan/stream") @limiter.limit("10/minute") async def stream_plan(request: Request, body: schemas.PlanRequest, db: Session = Depends(database.get_db)): """Server-Sent Events endpoint for /plan — streams steps as they become available so the frontend can render progress instead of waiting for the full response.""" async def generate(): try: check_budget(db, body.session_id) yield _sse({"type": "status", "message": "Analysing spec and scenario…"}) parsed_spec = json.loads(body.spec_content) if isinstance(body.spec_content, str) else body.spec_content credentials = body.user_credentials or {} if body.spec_id: spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() if spec and spec.login_credentials: credentials = decrypt_credentials(spec.login_credentials) yield _sse({"type": "status", "message": "Generating execution plan with AI…"}) llm = get_llm_for_task("plan_generation", db) plan = await asyncio.to_thread( generate_execution_plan, parsed_spec, body.base_url, body.scenario, credentials, llm ) log_usage(db, body.session_id, None, "plan", plan.token_usage.model_dump() if plan.token_usage else None) if body.spec_id: spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() if spec: spec.execution_plan = plan.model_dump()["execution_plan"] spec.scenario = body.scenario flag_modified(spec, "execution_plan") flag_modified(spec, "scenario") if body.session_id: session = db.query(database.SessionModel).filter( database.SessionModel.id == body.session_id ).first() if session: session.spec_id = spec.id session.spec_name = spec.name session.scenario = body.scenario session.base_url = body.base_url if plan.suggested_credentials and isinstance(plan.suggested_credentials, dict): current_creds = decrypt_credentials(dict(spec.login_credentials or {})) if any(k not in current_creds or current_creds[k] != v for k, v in plan.suggested_credentials.items()): current_creds.update(plan.suggested_credentials) spec.login_credentials = encrypt_credentials(current_creds) flag_modified(spec, "login_credentials") db.commit() # Stream each step individually so the UI can render them as they arrive for step in plan.execution_plan: yield _sse({"type": "step", "step": step.model_dump()}) await asyncio.sleep(0.04) yield _sse({ "type": "complete", "suggested_credentials": plan.suggested_credentials, "token_usage": plan.token_usage.model_dump() if plan.token_usage else None, }) except Exception as e: logger.error("SSE plan stream error: %s", traceback.format_exc()) yield _sse({"type": "error", "message": str(e)}) return StreamingResponse( generate(), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", "X-Accel-Buffering": "no", }, ) @router.post("/execute-step") @limiter.limit("30/minute") async def execute_step(request: Request, body: schemas.ExecuteStepRequest, db: Session = Depends(database.get_db)): try: check_budget(db, body.session_id) endpoint_str = body.step_definition.get("endpoint", "") actual_endpoint_spec = {"endpoint": endpoint_str} if body.spec_id: spec_doc = db.query(database.SpecDocument).filter( database.SpecDocument.id == body.spec_id ).first() if spec_doc and spec_doc.parsed_json: full_spec = spec_doc.parsed_json paths = full_spec.get("paths", {}) parts = endpoint_str.split(" ", 1) path_only = parts[1] if len(parts) == 2 else parts[0] matched_path = path_only if path_only in paths else next( (p for p in paths if _match_path(p, path_only)), None ) if matched_path: raw_spec = {matched_path: paths[matched_path]} resolved = _resolve_refs(raw_spec, full_spec) path_data = resolved.get(matched_path, {}) for method in path_data: if isinstance(path_data[method], dict): path_data[method].pop("responses", None) actual_endpoint_spec = _shorthand_spec(_prune_spec(resolved)) else: logger.warning("Path match failed for %s, falling back", path_only) actual_endpoint_spec = {"endpoint": endpoint_str, "warning": "Full schema match failed"} rich_assets = AssetService.get_ai_context(db, body.spec_id) effective_credentials = body.user_credentials if not effective_credentials and body.spec_id: spec = db.query(database.SpecDocument).filter( database.SpecDocument.id == body.spec_id ).first() if spec and spec.login_credentials: effective_credentials = decrypt_credentials(spec.login_credentials) logger.info("Using stored credentials for spec '%s'", spec.name) execution_context = dict(body.context_variables or {}) step_llm = get_llm_for_task("step_execution", db) req_plan, executor_token_usage = await asyncio.to_thread( prepare_step_request, endpoint_spec=actual_endpoint_spec, base_url=body.base_url, scenario="", step_definition=body.step_definition, context_variables=execution_context, user_credentials=effective_credentials, available_assets=rich_assets, llm=step_llm, ) # Guard: detect sentinel values that mean the AI couldn't resolve a real ID _MISSING_PATTERN = re.compile(r"__MISSING__(\w+)__") missing_in_url = _MISSING_PATTERN.findall(req_plan.request.url) missing_in_body = _MISSING_PATTERN.findall(json.dumps(req_plan.request.body or {})) all_missing = list(set(missing_in_url + missing_in_body)) if all_missing: raise HTTPException( status_code=400, detail={ "error": "missing_context_keys", "message": f"Step requires real IDs that are not in context: {all_missing}. Please provide them manually before executing this step.", "missing_keys": all_missing, }, ) url = req_plan.request.url.strip() if url.startswith("/"): url = body.base_url.rstrip("/") + url logger.info("[ROUTE] Relative URL resolved to: %s", url) elif _is_placeholder(url): parsed = urlparse(url) path = parsed.path or "/" url = body.base_url.rstrip("/") + path if parsed.query: url += "?" + parsed.query logger.info("[ROUTE] Placeholder URL replaced : %s", url) elif not url.startswith("http"): url = f"https://{url}" logger.info("[ROUTE] Scheme-prefixed URL : %s", url) url = _strip_phantom_path(url, body.base_url) # Merge query params from the URL and from the LLM's query_params dict into # a single authoritative dict, then strip the query string from the URL. parsed = urlparse(url) merged_query = dict(parse_qsl(parsed.query)) merged_query.update(req_plan.request.query_params or {}) url_no_query = url.split("?")[0] logger.info("[ROUTE] URL (no query) : %s", url_no_query) logger.info("[ROUTE] Merged query : %s", merged_query) # PHP single-file APIs that read the action from php://input instead of $_GET: # inject action into the JSON body for mutating methods only. GET requests # pass action via query string ($_GET) — adding a body to GET is non-standard # and may confuse the server or proxies. _method_upper = req_plan.request.method.strip().upper() if url_no_query.endswith(".php") and "action" in merged_query and _method_upper != "GET": _action = merged_query["action"] if isinstance(req_plan.request.body, dict): req_plan.request.body.setdefault("action", _action) elif req_plan.request.body is None: req_plan.request.body = {"action": _action} # Resolve IP once to block private/internal addresses (SSRF guard). # We use the original hostname URL for the actual request so TLS certificate # verification works correctly against the hostname (not the raw IP). resolve_safe_ip(url) async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client: req_headers = {k: v for k, v in req_plan.request.headers.items() if k.lower() != "cookie"} # Browser User-Agent to pass WAF/Cloudflare checks. if not any(k.lower() == "user-agent" for k in req_headers): req_headers["User-Agent"] = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/120.0.0.0 Safari/537.36" ) if not any(k.lower() == "accept" for k in req_headers): req_headers["Accept"] = "application/json" if not any(k.lower() == "accept-language" for k in req_headers): req_headers["Accept-Language"] = "en-US,en;q=0.9" # Embed query params directly into the URL string — same approach as # the Node.js test script which hardcodes ?action=me in the URL literal. final_url = ( f"{url_no_query}?{urlencode(merged_query)}" if merged_query else url_no_query ) http_req = client.build_request( method=req_plan.request.method.strip().upper(), url=final_url, headers=req_headers, data=req_plan.request.form_data if req_plan.request.form_data else None, json=req_plan.request.body if req_plan.request.body and not req_plan.request.form_data else None, ) # Log the ACTUAL URL and headers httpx will send — check backend console logger.info("="*70) logger.info("[ROUTE] >>> OUTGOING %s %s", http_req.method, http_req.url) logger.info("[ROUTE] >>> HEADERS %s", dict(http_req.headers)) if req_plan.request.body: logger.info("[ROUTE] >>> BODY %s", json.dumps(req_plan.request.body)) if req_plan.request.form_data: logger.info("[ROUTE] >>> FORMDATA %s", json.dumps(req_plan.request.form_data)) logger.info("="*70) files_payload = {} opened_files = [] try: if req_plan.request.files: req_headers = { k: v for k, v in req_headers.items() if k.lower() != "content-type" } for field_name, file_name in req_plan.request.files.items(): asset = db.query(database.Asset).filter(database.Asset.name == file_name).first() if asset: full_path = storage.get_full_path(asset.file_path) if os.path.exists(full_path): f = open(full_path, "rb") opened_files.append(f) files_payload[field_name] = (asset.name, f, asset.file_type) else: logger.warning("Asset file not found on disk: %s", full_path) else: logger.warning("Asset not found in Media Hub: %s", file_name) logger.info("Executing %s %s", http_req.method, http_req.url) try: if files_payload: response = await client.request( method=req_plan.request.method.strip().upper(), url=final_url, headers=req_headers, data=req_plan.request.form_data if req_plan.request.form_data else None, files=files_payload, ) else: response = await client.send(http_req) except httpx.TimeoutException: raise HTTPException( status_code=504, detail=f"Request to {url} timed out after {120}s. Check the API server is reachable.", ) except httpx.ConnectError as e: raise HTTPException( status_code=502, detail=f"Could not connect to {url}: {str(e)}", ) except Exception as e: raise HTTPException(status_code=500, detail=f"Request to {url} failed: {str(e)}") finally: for f in opened_files: f.close() body_text = response.text resp_data = { "status_code": response.status_code, "body": body_text, "headers": dict(response.headers), } resp_data["cookies"] = {} # Log incoming response — always visible in terminal logger.info("="*70) logger.info("[ROUTE] <<< INCOMING STATUS %s from %s", response.status_code, url_no_query) logger.info("[ROUTE] <<< RESPONSE HEADERS %s", dict(response.headers)) logger.info("[ROUTE] <<< RESPONSE BODY (first 1000 chars):\n%s", body_text[:1000]) logger.info("="*70) # Log a clear error line for 4xx/5xx so it's visible in the backend console if response.status_code >= 400: snippet = body_text[:300].replace("\n", " ") logger.warning( "[ROUTE] Step %s returned %s from %s: %s", req_plan.step, response.status_code, url, snippet, ) req_plan.request.url = url db.add(database.ExecutionHistory( session_id=body.session_id, step=req_plan.step, request_json=req_plan.request.model_dump(), response_json=resp_data, step_definition_json=body.step_definition, )) db.commit() log_usage(db, body.session_id, None, "execute_step", executor_token_usage) return { "executed_request": req_plan.request.model_dump(), "response": resp_data, "analysis_needed": req_plan.step, "token_usage": executor_token_usage, } except Exception as e: logger.error("Execution error: %s", traceback.format_exc()) raise HTTPException(status_code=500, detail=f"Execution failed: {str(e)}") @router.post("/analyze-response", response_model=schemas.AnalyzeResponse) async def analyze_response_endpoint(request: schemas.AnalyzeRequest, db: Session = Depends(database.get_db)): try: step_def = request.step_definition or {} analysis_llm = get_llm_for_task("response_analysis", db) analysis = await asyncio.to_thread( analyze_api_response, step=request.step, endpoint=step_def.get("endpoint", "Unknown"), purpose=step_def.get("purpose", "Unknown"), status_code=request.response_data.get("status_code", 0), response_body=request.response_data.get("body", ""), llm=analysis_llm, ) log_usage(db, request.session_id, None, "analyze", analysis.token_usage.model_dump() if analysis.token_usage else None) if analysis.extracted_variables: set_context_variables(db, request.session_id, analysis.extracted_variables) history = ( db.query(database.ExecutionHistory) .filter( database.ExecutionHistory.session_id == request.session_id, database.ExecutionHistory.step == request.step, ) .order_by(database.ExecutionHistory.id.desc()) .first() ) if history: history.analysis_json = analysis.model_dump() db.commit() return analysis except Exception as e: raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")