Spaces:
Runtime error
Runtime error
Download backend/app/api/routes/execution.py from Amitkasar/ai-api-testing: direct link, hf CLI and curl.
- Browser
- Download file 25.2 kB
-
https://huggingface.co/spaces/Amitkasar/ai-api-testing/resolve/main/backend/app/api/routes/execution.py
- Command line
-
hf download hf://spaces/Amitkasar/ai-api-testing/backend/app/api/routes/execution.py
-
curl -L -o execution.py https://huggingface.co/spaces/Amitkasar/ai-api-testing/resolve/main/backend/app/api/routes/execution.py
25.2 kB
| import asyncio | |
| import json | |
| import os | |
| import re | |
| import traceback | |
| from urllib.parse import parse_qsl, urlencode, urlparse | |
| import httpx | |
| from fastapi import APIRouter, Depends, HTTPException, Request | |
| from fastapi.responses import StreamingResponse | |
| from sqlalchemy.orm import Session | |
| from sqlalchemy.orm.attributes import flag_modified | |
| from backend import database, schemas | |
| from backend.core.crypto import decrypt_credentials, encrypt_credentials | |
| from backend.core.limiter import limiter | |
| from backend.core.logging import get_logger | |
| from backend.core.url_guard import resolve_safe_ip | |
| from backend.services.api_executor import prepare_step_request | |
| from backend.services.asset_service import AssetService, storage | |
| from backend.services.context_manager import ( | |
| set_context_variables, | |
| ) | |
| from backend.integrations.llm.resolver import get_llm_for_task | |
| from backend.services.llm_planner import generate_execution_plan | |
| from backend.services.response_analyzer import analyze_api_response | |
| from backend.services.usage_service import check_budget, log_usage | |
| router = APIRouter() | |
| logger = get_logger(__name__) | |
| PLACEHOLDER_HOSTS = {"api.example.com", "example.com", "<base_url>", "your-api.com"} | |
| def _resolve_refs(obj, full_spec, _resolving=None): | |
| if _resolving is None: | |
| _resolving = set() | |
| if isinstance(obj, dict): | |
| if "$ref" in obj: | |
| ref = obj["$ref"] | |
| if ref in _resolving: | |
| return {"$ref_loop": ref} | |
| _resolving = _resolving | {ref} | |
| parts = ref.lstrip("#/").split("/") | |
| target = full_spec | |
| for part in parts: | |
| target = target.get(part, {}) if isinstance(target, dict) else {} | |
| return _resolve_refs(target, full_spec, _resolving) | |
| return {k: _resolve_refs(v, full_spec, _resolving) for k, v in obj.items()} | |
| elif isinstance(obj, list): | |
| return [_resolve_refs(item, full_spec, _resolving) for item in obj] | |
| return obj | |
| def _shorthand_spec(obj): | |
| if isinstance(obj, dict): | |
| if "type" in obj: | |
| t = obj["type"] | |
| fmt = f":{obj['format']}" if "format" in obj else "" | |
| enum_vals = f"[{'|'.join(map(str, obj['enum']))}]" if "enum" in obj else "" | |
| if t == "object" and "properties" in obj: | |
| req_list = obj.get("required", []) | |
| props = [ | |
| f"{k}{'*' if k in req_list or v.get('nullable') == False else ''}({_shorthand_spec(v)})" | |
| for k, v in obj["properties"].items() | |
| ] | |
| return "{" + ",".join(props) + "}" | |
| if t == "array" and "items" in obj: | |
| return f"[{_shorthand_spec(obj['items'])}]" | |
| return f"{t}{fmt}{enum_vals}" | |
| return {k: _shorthand_spec(v) for k, v in obj.items()} | |
| return str(obj) | |
| def _prune_spec(obj): | |
| DROP_KEYS = {"description", "summary", "example", "examples", "format", "title", "default", "enum_descriptions"} | |
| if isinstance(obj, dict): | |
| return {k: _prune_spec(v) for k, v in obj.items() if k not in DROP_KEYS} | |
| elif isinstance(obj, list): | |
| return [_prune_spec(item) for item in obj] | |
| return obj | |
| def _match_path(path_template: str, actual_path: str) -> bool: | |
| pattern = re.sub(r'\{[^}]+\}', r'[^/]+', path_template) | |
| pattern = "^" + pattern.rstrip("/") + "/?$" | |
| return bool(re.match(pattern, actual_path.split("?")[0])) | |
| def _is_placeholder(url: str) -> bool: | |
| try: | |
| host = urlparse(url).hostname or "" | |
| return host.lower() in PLACEHOLDER_HOSTS or url.startswith("<BASE_URL>") | |
| except Exception: | |
| return False | |
| def _strip_phantom_path(url: str, base_url: str) -> str: | |
| """Remove path segments the LLM appended beyond a file-based base URL. | |
| Single-file APIs (e.g. admin_api.php) route via ?action=, not URL paths. | |
| When the OpenAPI spec defines /auth/me and the LLM appends that to | |
| admin_api.php, the result is admin_api.php/auth/me which 404s. | |
| We detect this by checking whether the base URL's last path segment has | |
| a file extension — if so, strip any extra segments the LLM added. | |
| """ | |
| try: | |
| base_parsed = urlparse(base_url if "://" in base_url else "https://" + base_url) | |
| base_file = base_parsed.path.rstrip("/").split("/")[-1] | |
| # Only act when base URL ends with a file (has an extension) | |
| if "." not in base_file: | |
| return url | |
| base_path = base_parsed.path.rstrip("/") | |
| url_parsed = urlparse(url) | |
| if url_parsed.path.startswith(base_path) and url_parsed.path != base_path: | |
| from urllib.parse import urlunparse | |
| # Preserve the query string — the caller's merged_query will merge it | |
| # with any explicit query_params from the LLM, so action= etc. survive. | |
| fixed = urlunparse(url_parsed._replace(path=base_path)) | |
| logger.info("Stripped phantom path '%s' → '%s'", url, fixed) | |
| return fixed | |
| except Exception: | |
| pass | |
| return url | |
| async def generate_plan(request: Request, body: schemas.PlanRequest, db: Session = Depends(database.get_db)): | |
| try: | |
| check_budget(db, body.session_id) | |
| parsed_spec = json.loads(body.spec_content) if isinstance(body.spec_content, str) else body.spec_content | |
| credentials = body.user_credentials or {} | |
| if body.spec_id: | |
| spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() | |
| if spec and spec.login_credentials: | |
| credentials = decrypt_credentials(spec.login_credentials) | |
| llm = get_llm_for_task("plan_generation", db) | |
| plan = await asyncio.to_thread(generate_execution_plan, parsed_spec, body.base_url, body.scenario, credentials, llm) | |
| log_usage(db, body.session_id, None, "plan", plan.token_usage.model_dump() if plan.token_usage else None) | |
| if body.spec_id: | |
| spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() | |
| if spec: | |
| spec.execution_plan = plan.model_dump()["execution_plan"] | |
| spec.scenario = body.scenario | |
| flag_modified(spec, "execution_plan") | |
| flag_modified(spec, "scenario") | |
| if body.session_id: | |
| session = db.query(database.SessionModel).filter( | |
| database.SessionModel.id == body.session_id | |
| ).first() | |
| if session: | |
| session.spec_id = spec.id | |
| session.spec_name = spec.name | |
| session.scenario = body.scenario | |
| session.base_url = body.base_url | |
| if plan.suggested_credentials and isinstance(plan.suggested_credentials, dict): | |
| current_creds = decrypt_credentials(dict(spec.login_credentials or {})) | |
| if any(k not in current_creds or current_creds[k] != v for k, v in plan.suggested_credentials.items()): | |
| current_creds.update(plan.suggested_credentials) | |
| spec.login_credentials = encrypt_credentials(current_creds) | |
| flag_modified(spec, "login_credentials") | |
| logger.info("AI suggested credentials merged for spec '%s'", spec.name) | |
| db.commit() | |
| return plan | |
| except Exception as e: | |
| raise HTTPException(status_code=500, detail=f"Planning failed: {str(e)}") | |
| def _sse(payload: dict) -> str: | |
| return f"data: {json.dumps(payload)}\n\n" | |
| async def stream_plan(request: Request, body: schemas.PlanRequest, db: Session = Depends(database.get_db)): | |
| """Server-Sent Events endpoint for /plan — streams steps as they become available so the | |
| frontend can render progress instead of waiting for the full response.""" | |
| async def generate(): | |
| try: | |
| check_budget(db, body.session_id) | |
| yield _sse({"type": "status", "message": "Analysing spec and scenario…"}) | |
| parsed_spec = json.loads(body.spec_content) if isinstance(body.spec_content, str) else body.spec_content | |
| credentials = body.user_credentials or {} | |
| if body.spec_id: | |
| spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() | |
| if spec and spec.login_credentials: | |
| credentials = decrypt_credentials(spec.login_credentials) | |
| yield _sse({"type": "status", "message": "Generating execution plan with AI…"}) | |
| llm = get_llm_for_task("plan_generation", db) | |
| plan = await asyncio.to_thread( | |
| generate_execution_plan, parsed_spec, body.base_url, body.scenario, credentials, llm | |
| ) | |
| log_usage(db, body.session_id, None, "plan", plan.token_usage.model_dump() if plan.token_usage else None) | |
| if body.spec_id: | |
| spec = db.query(database.SpecDocument).filter(database.SpecDocument.id == body.spec_id).first() | |
| if spec: | |
| spec.execution_plan = plan.model_dump()["execution_plan"] | |
| spec.scenario = body.scenario | |
| flag_modified(spec, "execution_plan") | |
| flag_modified(spec, "scenario") | |
| if body.session_id: | |
| session = db.query(database.SessionModel).filter( | |
| database.SessionModel.id == body.session_id | |
| ).first() | |
| if session: | |
| session.spec_id = spec.id | |
| session.spec_name = spec.name | |
| session.scenario = body.scenario | |
| session.base_url = body.base_url | |
| if plan.suggested_credentials and isinstance(plan.suggested_credentials, dict): | |
| current_creds = decrypt_credentials(dict(spec.login_credentials or {})) | |
| if any(k not in current_creds or current_creds[k] != v for k, v in plan.suggested_credentials.items()): | |
| current_creds.update(plan.suggested_credentials) | |
| spec.login_credentials = encrypt_credentials(current_creds) | |
| flag_modified(spec, "login_credentials") | |
| db.commit() | |
| # Stream each step individually so the UI can render them as they arrive | |
| for step in plan.execution_plan: | |
| yield _sse({"type": "step", "step": step.model_dump()}) | |
| await asyncio.sleep(0.04) | |
| yield _sse({ | |
| "type": "complete", | |
| "suggested_credentials": plan.suggested_credentials, | |
| "token_usage": plan.token_usage.model_dump() if plan.token_usage else None, | |
| }) | |
| except Exception as e: | |
| logger.error("SSE plan stream error: %s", traceback.format_exc()) | |
| yield _sse({"type": "error", "message": str(e)}) | |
| return StreamingResponse( | |
| generate(), | |
| media_type="text/event-stream", | |
| headers={ | |
| "Cache-Control": "no-cache", | |
| "X-Accel-Buffering": "no", | |
| }, | |
| ) | |
| async def execute_step(request: Request, body: schemas.ExecuteStepRequest, db: Session = Depends(database.get_db)): | |
| try: | |
| check_budget(db, body.session_id) | |
| endpoint_str = body.step_definition.get("endpoint", "") | |
| actual_endpoint_spec = {"endpoint": endpoint_str} | |
| if body.spec_id: | |
| spec_doc = db.query(database.SpecDocument).filter( | |
| database.SpecDocument.id == body.spec_id | |
| ).first() | |
| if spec_doc and spec_doc.parsed_json: | |
| full_spec = spec_doc.parsed_json | |
| paths = full_spec.get("paths", {}) | |
| parts = endpoint_str.split(" ", 1) | |
| path_only = parts[1] if len(parts) == 2 else parts[0] | |
| matched_path = path_only if path_only in paths else next( | |
| (p for p in paths if _match_path(p, path_only)), None | |
| ) | |
| if matched_path: | |
| raw_spec = {matched_path: paths[matched_path]} | |
| resolved = _resolve_refs(raw_spec, full_spec) | |
| path_data = resolved.get(matched_path, {}) | |
| for method in path_data: | |
| if isinstance(path_data[method], dict): | |
| path_data[method].pop("responses", None) | |
| actual_endpoint_spec = _shorthand_spec(_prune_spec(resolved)) | |
| else: | |
| logger.warning("Path match failed for %s, falling back", path_only) | |
| actual_endpoint_spec = {"endpoint": endpoint_str, "warning": "Full schema match failed"} | |
| rich_assets = AssetService.get_ai_context(db, body.spec_id) | |
| effective_credentials = body.user_credentials | |
| if not effective_credentials and body.spec_id: | |
| spec = db.query(database.SpecDocument).filter( | |
| database.SpecDocument.id == body.spec_id | |
| ).first() | |
| if spec and spec.login_credentials: | |
| effective_credentials = decrypt_credentials(spec.login_credentials) | |
| logger.info("Using stored credentials for spec '%s'", spec.name) | |
| execution_context = dict(body.context_variables or {}) | |
| step_llm = get_llm_for_task("step_execution", db) | |
| req_plan, executor_token_usage = await asyncio.to_thread( | |
| prepare_step_request, | |
| endpoint_spec=actual_endpoint_spec, | |
| base_url=body.base_url, | |
| scenario="", | |
| step_definition=body.step_definition, | |
| context_variables=execution_context, | |
| user_credentials=effective_credentials, | |
| available_assets=rich_assets, | |
| llm=step_llm, | |
| ) | |
| # Guard: detect sentinel values that mean the AI couldn't resolve a real ID | |
| _MISSING_PATTERN = re.compile(r"__MISSING__(\w+)__") | |
| missing_in_url = _MISSING_PATTERN.findall(req_plan.request.url) | |
| missing_in_body = _MISSING_PATTERN.findall(json.dumps(req_plan.request.body or {})) | |
| all_missing = list(set(missing_in_url + missing_in_body)) | |
| if all_missing: | |
| raise HTTPException( | |
| status_code=400, | |
| detail={ | |
| "error": "missing_context_keys", | |
| "message": f"Step requires real IDs that are not in context: {all_missing}. Please provide them manually before executing this step.", | |
| "missing_keys": all_missing, | |
| }, | |
| ) | |
| url = req_plan.request.url.strip() | |
| if url.startswith("/"): | |
| url = body.base_url.rstrip("/") + url | |
| logger.info("[ROUTE] Relative URL resolved to: %s", url) | |
| elif _is_placeholder(url): | |
| parsed = urlparse(url) | |
| path = parsed.path or "/" | |
| url = body.base_url.rstrip("/") + path | |
| if parsed.query: | |
| url += "?" + parsed.query | |
| logger.info("[ROUTE] Placeholder URL replaced : %s", url) | |
| elif not url.startswith("http"): | |
| url = f"https://{url}" | |
| logger.info("[ROUTE] Scheme-prefixed URL : %s", url) | |
| url = _strip_phantom_path(url, body.base_url) | |
| # Merge query params from the URL and from the LLM's query_params dict into | |
| # a single authoritative dict, then strip the query string from the URL. | |
| parsed = urlparse(url) | |
| merged_query = dict(parse_qsl(parsed.query)) | |
| merged_query.update(req_plan.request.query_params or {}) | |
| url_no_query = url.split("?")[0] | |
| logger.info("[ROUTE] URL (no query) : %s", url_no_query) | |
| logger.info("[ROUTE] Merged query : %s", merged_query) | |
| # PHP single-file APIs that read the action from php://input instead of $_GET: | |
| # inject action into the JSON body for mutating methods only. GET requests | |
| # pass action via query string ($_GET) — adding a body to GET is non-standard | |
| # and may confuse the server or proxies. | |
| _method_upper = req_plan.request.method.strip().upper() | |
| if url_no_query.endswith(".php") and "action" in merged_query and _method_upper != "GET": | |
| _action = merged_query["action"] | |
| if isinstance(req_plan.request.body, dict): | |
| req_plan.request.body.setdefault("action", _action) | |
| elif req_plan.request.body is None: | |
| req_plan.request.body = {"action": _action} | |
| # Resolve IP once to block private/internal addresses (SSRF guard). | |
| # We use the original hostname URL for the actual request so TLS certificate | |
| # verification works correctly against the hostname (not the raw IP). | |
| resolve_safe_ip(url) | |
| async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client: | |
| req_headers = {k: v for k, v in req_plan.request.headers.items() if k.lower() != "cookie"} | |
| # Browser User-Agent to pass WAF/Cloudflare checks. | |
| if not any(k.lower() == "user-agent" for k in req_headers): | |
| req_headers["User-Agent"] = ( | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " | |
| "AppleWebKit/537.36 (KHTML, like Gecko) " | |
| "Chrome/120.0.0.0 Safari/537.36" | |
| ) | |
| if not any(k.lower() == "accept" for k in req_headers): | |
| req_headers["Accept"] = "application/json" | |
| if not any(k.lower() == "accept-language" for k in req_headers): | |
| req_headers["Accept-Language"] = "en-US,en;q=0.9" | |
| # Embed query params directly into the URL string — same approach as | |
| # the Node.js test script which hardcodes ?action=me in the URL literal. | |
| final_url = ( | |
| f"{url_no_query}?{urlencode(merged_query)}" if merged_query else url_no_query | |
| ) | |
| http_req = client.build_request( | |
| method=req_plan.request.method.strip().upper(), | |
| url=final_url, | |
| headers=req_headers, | |
| data=req_plan.request.form_data if req_plan.request.form_data else None, | |
| json=req_plan.request.body if req_plan.request.body and not req_plan.request.form_data else None, | |
| ) | |
| # Log the ACTUAL URL and headers httpx will send — check backend console | |
| logger.info("="*70) | |
| logger.info("[ROUTE] >>> OUTGOING %s %s", http_req.method, http_req.url) | |
| logger.info("[ROUTE] >>> HEADERS %s", dict(http_req.headers)) | |
| if req_plan.request.body: | |
| logger.info("[ROUTE] >>> BODY %s", json.dumps(req_plan.request.body)) | |
| if req_plan.request.form_data: | |
| logger.info("[ROUTE] >>> FORMDATA %s", json.dumps(req_plan.request.form_data)) | |
| logger.info("="*70) | |
| files_payload = {} | |
| opened_files = [] | |
| try: | |
| if req_plan.request.files: | |
| req_headers = { | |
| k: v for k, v in req_headers.items() if k.lower() != "content-type" | |
| } | |
| for field_name, file_name in req_plan.request.files.items(): | |
| asset = db.query(database.Asset).filter(database.Asset.name == file_name).first() | |
| if asset: | |
| full_path = storage.get_full_path(asset.file_path) | |
| if os.path.exists(full_path): | |
| f = open(full_path, "rb") | |
| opened_files.append(f) | |
| files_payload[field_name] = (asset.name, f, asset.file_type) | |
| else: | |
| logger.warning("Asset file not found on disk: %s", full_path) | |
| else: | |
| logger.warning("Asset not found in Media Hub: %s", file_name) | |
| logger.info("Executing %s %s", http_req.method, http_req.url) | |
| try: | |
| if files_payload: | |
| response = await client.request( | |
| method=req_plan.request.method.strip().upper(), | |
| url=final_url, | |
| headers=req_headers, | |
| data=req_plan.request.form_data if req_plan.request.form_data else None, | |
| files=files_payload, | |
| ) | |
| else: | |
| response = await client.send(http_req) | |
| except httpx.TimeoutException: | |
| raise HTTPException( | |
| status_code=504, | |
| detail=f"Request to {url} timed out after {120}s. Check the API server is reachable.", | |
| ) | |
| except httpx.ConnectError as e: | |
| raise HTTPException( | |
| status_code=502, | |
| detail=f"Could not connect to {url}: {str(e)}", | |
| ) | |
| except Exception as e: | |
| raise HTTPException(status_code=500, detail=f"Request to {url} failed: {str(e)}") | |
| finally: | |
| for f in opened_files: | |
| f.close() | |
| body_text = response.text | |
| resp_data = { | |
| "status_code": response.status_code, | |
| "body": body_text, | |
| "headers": dict(response.headers), | |
| } | |
| resp_data["cookies"] = {} | |
| # Log incoming response — always visible in terminal | |
| logger.info("="*70) | |
| logger.info("[ROUTE] <<< INCOMING STATUS %s from %s", response.status_code, url_no_query) | |
| logger.info("[ROUTE] <<< RESPONSE HEADERS %s", dict(response.headers)) | |
| logger.info("[ROUTE] <<< RESPONSE BODY (first 1000 chars):\n%s", body_text[:1000]) | |
| logger.info("="*70) | |
| # Log a clear error line for 4xx/5xx so it's visible in the backend console | |
| if response.status_code >= 400: | |
| snippet = body_text[:300].replace("\n", " ") | |
| logger.warning( | |
| "[ROUTE] Step %s returned %s from %s: %s", | |
| req_plan.step, response.status_code, url, snippet, | |
| ) | |
| req_plan.request.url = url | |
| db.add(database.ExecutionHistory( | |
| session_id=body.session_id, | |
| step=req_plan.step, | |
| request_json=req_plan.request.model_dump(), | |
| response_json=resp_data, | |
| step_definition_json=body.step_definition, | |
| )) | |
| db.commit() | |
| log_usage(db, body.session_id, None, "execute_step", executor_token_usage) | |
| return { | |
| "executed_request": req_plan.request.model_dump(), | |
| "response": resp_data, | |
| "analysis_needed": req_plan.step, | |
| "token_usage": executor_token_usage, | |
| } | |
| except Exception as e: | |
| logger.error("Execution error: %s", traceback.format_exc()) | |
| raise HTTPException(status_code=500, detail=f"Execution failed: {str(e)}") | |
| async def analyze_response_endpoint(request: schemas.AnalyzeRequest, db: Session = Depends(database.get_db)): | |
| try: | |
| step_def = request.step_definition or {} | |
| analysis_llm = get_llm_for_task("response_analysis", db) | |
| analysis = await asyncio.to_thread( | |
| analyze_api_response, | |
| step=request.step, | |
| endpoint=step_def.get("endpoint", "Unknown"), | |
| purpose=step_def.get("purpose", "Unknown"), | |
| status_code=request.response_data.get("status_code", 0), | |
| response_body=request.response_data.get("body", ""), | |
| llm=analysis_llm, | |
| ) | |
| log_usage(db, request.session_id, None, "analyze", analysis.token_usage.model_dump() if analysis.token_usage else None) | |
| if analysis.extracted_variables: | |
| set_context_variables(db, request.session_id, analysis.extracted_variables) | |
| history = ( | |
| db.query(database.ExecutionHistory) | |
| .filter( | |
| database.ExecutionHistory.session_id == request.session_id, | |
| database.ExecutionHistory.step == request.step, | |
| ) | |
| .order_by(database.ExecutionHistory.id.desc()) | |
| .first() | |
| ) | |
| if history: | |
| history.analysis_json = analysis.model_dump() | |
| db.commit() | |
| return analysis | |
| except Exception as e: | |
| raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}") | |