Spaces:
Runtime error
Security & reliability: Phase 1–3 fixes (Items 1–15)
Browse files- git rm --cached tracked binary uploads from documents/
- Add .env.example with placeholder values
- Extract shared Limiter to backend/core/limiter.py (Item 14)
- Tighten CORS: restrict allow_methods and allow_headers (Item 7)
- Fix traceback import inside except in chat.py; add logger (Item 13)
- Fix DNS rebinding SSRF: resolve IP once via resolve_safe_ip(), pin to
resolved IP with Host header so httpx cannot re-resolve (Item 4)
- Fix mutable default args in prepare_step_request (Item 10)
- Wrap prepare_step_request in asyncio.to_thread() (Item 8)
- Add LLM parse retry with AppError after 2 failures in api_executor (Item 9)
- Replace print() with logger calls in database.py and chat_assistant.py (Item 12)
- Increase response truncation to 4000 chars, JSON-aware, configurable (Item 15)
- Add RESPONSE_TRUNCATION_LIMIT to settings (Item 15)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- .env.example +12 -0
- Plans/plan 3.md +171 -0
- backend/app/api/routes/chat.py +5 -5
- backend/app/api/routes/execution.py +21 -13
- backend/core/config.py +1 -0
- backend/core/limiter.py +4 -0
- backend/core/url_guard.py +14 -3
- backend/database.py +9 -7
- backend/main.py +4 -7
- backend/services/api_executor.py +23 -11
- backend/services/chat_assistant.py +5 -2
- backend/services/response_analyzer.py +9 -3
- backend/uploads/documents/a1f57578-141b-4520-963b-c1023c93bfa6.png +0 -3
- backend/uploads/documents/be351df0-6bd3-46dc-b154-805f9ec8d127.jpg +0 -3
- backend/uploads/documents/dd08cd23-5485-4ab7-8c05-8ee9d9e345da.jpg +0 -3
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MISTRAL_API_KEY=your_mistral_api_key_here
|
| 2 |
+
MISTRAL_MODEL=mistral-large-latest
|
| 3 |
+
|
| 4 |
+
DATABASE_URL=postgresql://postgres:YOUR_PASSWORD@db.YOUR_PROJECT.supabase.co:5432/postgres
|
| 5 |
+
SUPABASE_URL=https://YOUR_PROJECT.supabase.co
|
| 6 |
+
SUPABASE_KEY=your_supabase_anon_key_here
|
| 7 |
+
SUPABASE_STORAGE_BUCKET=assets
|
| 8 |
+
SUPABASE_SECRET_KEY=your_supabase_secret_key_here
|
| 9 |
+
SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key_here
|
| 10 |
+
|
| 11 |
+
VITE_BACKEND_URL=http://localhost:8001
|
| 12 |
+
VITE_API_URL=http://localhost:8001
|
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Phase 1 – Emergency Fixes (Day 1)
|
| 2 |
+
|
| 3 |
+
1. Remove binary uploads from git
|
| 4 |
+
|
| 5 |
+
* 3 image files are tracked in git: a1f57578.png, be351df0.jpg, dd08cd23.jpg inside backend/uploads/documents/.
|
| 6 |
+
* Run: git rm --cached backend/uploads/documents/*.png backend/uploads/documents/*.jpg
|
| 7 |
+
* Commit the removal. .gitkeep files stay, images are gone from history.
|
| 8 |
+
* .gitignore already has the correct upload patterns — no change needed there.
|
| 9 |
+
* Files: backend/uploads/documents/
|
| 10 |
+
|
| 11 |
+
2. Create .env.example
|
| 12 |
+
|
| 13 |
+
* .env is already in .gitignore and was NEVER committed to git — keys are safe, no history cleanup needed.
|
| 14 |
+
* Create .env.example with placeholder values so new developers know what to set.
|
| 15 |
+
* Still rotate the real Supabase keys as best practice (they exist on disk).
|
| 16 |
+
* Files: .env.example (new)
|
| 17 |
+
|
| 18 |
+
Phase 2 – Security Hardening (Week 1)
|
| 19 |
+
|
| 20 |
+
3. Add authentication to all endpoints
|
| 21 |
+
|
| 22 |
+
* All API routes are currently public with no authentication.
|
| 23 |
+
* Add FastAPI dependency: verify_token(token: str = Depends(oauth2_scheme)).
|
| 24 |
+
* Apply it globally in main.py or per-router using dependencies=[Depends(verify_token)].
|
| 25 |
+
* Use JWT (PyJWT) or Supabase Auth.
|
| 26 |
+
* Add login/logout endpoints that issue short-lived JWTs.
|
| 27 |
+
* On frontend, store token in memory (not localStorage) and send it as Authorization: Bearer <token>.
|
| 28 |
+
* Files: backend/core/auth.py (new), backend/main.py, frontend-react/src/services/api.js
|
| 29 |
+
|
| 30 |
+
4. Fix DNS rebinding SSRF bypass
|
| 31 |
+
|
| 32 |
+
* url_guard.py resolves IP once in assert_safe_url() and checks it — good start.
|
| 33 |
+
* But httpx re-resolves the hostname on its own, so an attacker can change DNS between the check and the request.
|
| 34 |
+
* Fix: resolve IP once, block if private, then pass the resolved IP directly to httpx with the original Host header set manually.
|
| 35 |
+
* Files: backend/core/url_guard.py, backend/app/api/routes/execution.py
|
| 36 |
+
|
| 37 |
+
5. Encrypt stored credentials
|
| 38 |
+
|
| 39 |
+
* login_credentials currently stores username/password in plain text JSON.
|
| 40 |
+
* Install cryptography: pip install cryptography.
|
| 41 |
+
* Add ENCRYPTION_KEY to .env (use Fernet.generate_key()).
|
| 42 |
+
* Create encrypt_json() and decrypt_json() using Fernet.
|
| 43 |
+
* Encrypt when saving, decrypt when reading.
|
| 44 |
+
* Files: backend/core/crypto.py (new), backend/app/api/routes/specs.py, backend/app/api/routes/execution.py
|
| 45 |
+
|
| 46 |
+
6. Tie sessions to authenticated users
|
| 47 |
+
|
| 48 |
+
* Sessions have no user connection — anyone with UUID can access them.
|
| 49 |
+
* Add user_id column to SessionModel and ChatSession.
|
| 50 |
+
* Inject current_user using Depends(verify_token).
|
| 51 |
+
* Filter DB queries by user_id.
|
| 52 |
+
* Implement after item 3 (auth) is done.
|
| 53 |
+
* Files: backend/database.py, backend/app/api/routes/sessions.py, backend/app/api/routes/chat.py
|
| 54 |
+
|
| 55 |
+
7. Tighten CORS policy
|
| 56 |
+
|
| 57 |
+
* Current config allows everything (*) for methods and headers.
|
| 58 |
+
* Change allow_methods to ['GET','POST','PUT','PATCH','DELETE'].
|
| 59 |
+
* Change allow_headers to ['Content-Type','Authorization'].
|
| 60 |
+
* Keep allow_origins from settings.
|
| 61 |
+
* Files: backend/main.py
|
| 62 |
+
|
| 63 |
+
Phase 3 – Backend Reliability (Week 2)
|
| 64 |
+
|
| 65 |
+
8. Fix event loop blocking in execute_step
|
| 66 |
+
|
| 67 |
+
* prepare_step_request() is sync but used in async route.
|
| 68 |
+
* Wrap using asyncio.to_thread().
|
| 69 |
+
* Import asyncio.
|
| 70 |
+
* Do same for generate_execution_plan().
|
| 71 |
+
* Files: backend/app/api/routes/execution.py
|
| 72 |
+
|
| 73 |
+
9. Add retry logic to api_executor
|
| 74 |
+
|
| 75 |
+
* No retry when LLM parsing fails.
|
| 76 |
+
* Wrap parser.parse() in try/except.
|
| 77 |
+
* Retry once if it fails.
|
| 78 |
+
* Log error with logger.warning().
|
| 79 |
+
* After 2 failures, raise AppError.
|
| 80 |
+
* Files: backend/services/api_executor.py
|
| 81 |
+
|
| 82 |
+
10. Fix mutable default arguments
|
| 83 |
+
|
| 84 |
+
* Default dicts (user_credentials={}, available_assets={}) in prepare_step_request() are shared across calls — real bug risk.
|
| 85 |
+
* Change defaults to None.
|
| 86 |
+
* Inside function: assign empty dict if None.
|
| 87 |
+
* Files: backend/services/api_executor.py
|
| 88 |
+
|
| 89 |
+
11. Migrate from create_all() to Alembic
|
| 90 |
+
|
| 91 |
+
* create_all() runs twice and has no schema history.
|
| 92 |
+
* Install Alembic and initialize it.
|
| 93 |
+
* Configure env.py with Base.metadata and DB URL.
|
| 94 |
+
* Generate migration and upgrade.
|
| 95 |
+
* Remove create_all() later.
|
| 96 |
+
* Files: alembic/, backend/database.py, backend/main.py
|
| 97 |
+
|
| 98 |
+
12. Fix DB host leak and debug prints
|
| 99 |
+
|
| 100 |
+
* print() leaks DB host and bypasses logging.
|
| 101 |
+
* Replace with logger.info() or logger.error().
|
| 102 |
+
* Remove debug prints.
|
| 103 |
+
* Files: backend/database.py, backend/services/chat_assistant.py, backend/services/api_executor.py
|
| 104 |
+
|
| 105 |
+
13. Fix import inside except block
|
| 106 |
+
|
| 107 |
+
* import traceback is inside the except block in chat.py at line 91.
|
| 108 |
+
* execution.py is already correct (traceback imported at top).
|
| 109 |
+
* Move it to the top of chat.py.
|
| 110 |
+
* Files: backend/app/api/routes/chat.py
|
| 111 |
+
|
| 112 |
+
14. Share the rate limiter across routers
|
| 113 |
+
|
| 114 |
+
* main.py already creates a shared limiter on app.state.limiter — good.
|
| 115 |
+
* But chat.py (line 14) and execution.py (line 25) each still create their own separate Limiter() instance.
|
| 116 |
+
* Remove the local Limiter() from both routers.
|
| 117 |
+
* In rate-limited endpoints, use request.app.state.limiter or pass the shared limiter via a dependency.
|
| 118 |
+
* Files: backend/app/api/routes/chat.py, backend/app/api/routes/execution.py
|
| 119 |
+
|
| 120 |
+
15. Increase response truncation limit
|
| 121 |
+
|
| 122 |
+
* response_analyzer cuts response to 1000 chars.
|
| 123 |
+
* Increase to 4000 chars.
|
| 124 |
+
* Make configurable via settings.
|
| 125 |
+
* For JSON, parse instead of slicing.
|
| 126 |
+
* Files: backend/services/response_analyzer.py, backend/core/config.py
|
| 127 |
+
|
| 128 |
+
Phase 4 – Architecture Improvements (Week 3–4)
|
| 129 |
+
|
| 130 |
+
16. Add file type validation (magic bytes)
|
| 131 |
+
|
| 132 |
+
* Files are validated only by extension.
|
| 133 |
+
* Install python-magic.
|
| 134 |
+
* Detect MIME type after upload.
|
| 135 |
+
* Compare with allowed types.
|
| 136 |
+
* Delete and reject if mismatch.
|
| 137 |
+
* Files: backend/services/asset_service.py, requirements.txt
|
| 138 |
+
|
| 139 |
+
17. Add LLM cost tracking and budget cap
|
| 140 |
+
|
| 141 |
+
* No tracking of token usage or cost.
|
| 142 |
+
* Create LLMUsage table.
|
| 143 |
+
* Store token usage for each call.
|
| 144 |
+
* Add /usage endpoint.
|
| 145 |
+
* Add max token limit per session.
|
| 146 |
+
* Files: backend/database.py, backend/services/, backend/app/api/routes/, backend/core/config.py
|
| 147 |
+
|
| 148 |
+
18. Abstract LLM client for provider fallback
|
| 149 |
+
|
| 150 |
+
* If Mistral fails, system has no backup.
|
| 151 |
+
* Create BaseLLMClient interface.
|
| 152 |
+
* Implement Mistral and another provider.
|
| 153 |
+
* Add config for primary and fallback.
|
| 154 |
+
* Retry with fallback on failure.
|
| 155 |
+
* Files: backend/integrations/llm/base.py, backend/integrations/llm/client.py, backend/core/config.py
|
| 156 |
+
|
| 157 |
+
19. Break up App.jsx (946 lines)
|
| 158 |
+
|
| 159 |
+
* App.jsx is 946 lines and handles too much logic.
|
| 160 |
+
* frontend-react/src/services/api.js already exists — move all API calls there.
|
| 161 |
+
* Separate component files already exist in src/components/ — keep and expand.
|
| 162 |
+
* Use react-router-dom (already installed) for page routing.
|
| 163 |
+
* Keep App.jsx only for routing and layout.
|
| 164 |
+
* Files: frontend-react/src/App.jsx, frontend-react/src/pages/, frontend-react/src/services/api.js
|
| 165 |
+
|
| 166 |
+
---
|
| 167 |
+
|
| 168 |
+
Implementation Order (recommended)
|
| 169 |
+
|
| 170 |
+
Phase 1 first, then within Phase 2–3 follow this sequence:
|
| 171 |
+
Item 1 → Item 2 → Item 7 → Item 13 → Item 14 → Item 10 → Item 8 → Items 9/12/15 → Item 4 → Items 3/5/6 (together) → Phase 4
|
|
@@ -1,17 +1,18 @@
|
|
| 1 |
import datetime
|
|
|
|
| 2 |
import uuid
|
| 3 |
from typing import Optional
|
| 4 |
|
| 5 |
from fastapi import APIRouter, Depends, HTTPException, Request
|
| 6 |
-
from slowapi import Limiter
|
| 7 |
-
from slowapi.util import get_remote_address
|
| 8 |
from sqlalchemy.orm import Session
|
| 9 |
|
| 10 |
from backend import database, schemas
|
|
|
|
|
|
|
| 11 |
from backend.services.chat_assistant import process_message as chat_process_message
|
| 12 |
|
| 13 |
router = APIRouter()
|
| 14 |
-
|
| 15 |
|
| 16 |
|
| 17 |
@router.post("/chat", response_model=schemas.ChatResponse)
|
|
@@ -88,8 +89,7 @@ def chat_endpoint(request: Request, body: schemas.ChatRequest, db: Session = Dep
|
|
| 88 |
explanation=ai_result.get("explanation"),
|
| 89 |
)
|
| 90 |
except Exception as e:
|
| 91 |
-
|
| 92 |
-
traceback.print_exc()
|
| 93 |
raise HTTPException(status_code=500, detail=f"Chat failed: {str(e)}")
|
| 94 |
|
| 95 |
|
|
|
|
| 1 |
import datetime
|
| 2 |
+
import traceback
|
| 3 |
import uuid
|
| 4 |
from typing import Optional
|
| 5 |
|
| 6 |
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
|
|
|
|
|
| 7 |
from sqlalchemy.orm import Session
|
| 8 |
|
| 9 |
from backend import database, schemas
|
| 10 |
+
from backend.core.limiter import limiter
|
| 11 |
+
from backend.core.logging import get_logger
|
| 12 |
from backend.services.chat_assistant import process_message as chat_process_message
|
| 13 |
|
| 14 |
router = APIRouter()
|
| 15 |
+
logger = get_logger(__name__)
|
| 16 |
|
| 17 |
|
| 18 |
@router.post("/chat", response_model=schemas.ChatResponse)
|
|
|
|
| 89 |
explanation=ai_result.get("explanation"),
|
| 90 |
)
|
| 91 |
except Exception as e:
|
| 92 |
+
logger.error("Chat failed: %s", traceback.format_exc())
|
|
|
|
| 93 |
raise HTTPException(status_code=500, detail=f"Chat failed: {str(e)}")
|
| 94 |
|
| 95 |
|
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
import json
|
| 2 |
import os
|
| 3 |
import re
|
|
@@ -6,14 +7,13 @@ from urllib.parse import parse_qsl, urlparse
|
|
| 6 |
|
| 7 |
import httpx
|
| 8 |
from fastapi import APIRouter, Depends, HTTPException, Request
|
| 9 |
-
from slowapi import Limiter
|
| 10 |
-
from slowapi.util import get_remote_address
|
| 11 |
from sqlalchemy.orm import Session
|
| 12 |
from sqlalchemy.orm.attributes import flag_modified
|
| 13 |
|
| 14 |
from backend import database, schemas
|
|
|
|
| 15 |
from backend.core.logging import get_logger
|
| 16 |
-
from backend.core.url_guard import
|
| 17 |
from backend.services.api_executor import prepare_step_request
|
| 18 |
from backend.services.asset_service import AssetService, storage
|
| 19 |
from backend.services.context_manager import set_context_variables
|
|
@@ -22,7 +22,6 @@ from backend.services.response_analyzer import analyze_api_response
|
|
| 22 |
|
| 23 |
router = APIRouter()
|
| 24 |
logger = get_logger(__name__)
|
| 25 |
-
limiter = Limiter(key_func=get_remote_address)
|
| 26 |
|
| 27 |
PLACEHOLDER_HOSTS = {"api.example.com", "example.com", "<base_url>", "your-api.com"}
|
| 28 |
|
|
@@ -181,7 +180,8 @@ async def execute_step(request: Request, body: schemas.ExecuteStepRequest, db: S
|
|
| 181 |
effective_credentials = spec.login_credentials
|
| 182 |
logger.info("Using stored credentials for spec '%s'", spec.name)
|
| 183 |
|
| 184 |
-
req_plan, executor_token_usage =
|
|
|
|
| 185 |
endpoint_spec=actual_endpoint_spec,
|
| 186 |
base_url=body.base_url,
|
| 187 |
scenario="",
|
|
@@ -210,14 +210,22 @@ async def execute_step(request: Request, body: schemas.ExecuteStepRequest, db: S
|
|
| 210 |
for k in [k for k in req_plan.request.query_params if k in url_keys]:
|
| 211 |
del req_plan.request.query_params[k]
|
| 212 |
|
| 213 |
-
#
|
| 214 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
|
| 216 |
async with httpx.AsyncClient(timeout=120.0) as client:
|
|
|
|
|
|
|
|
|
|
| 217 |
http_req = client.build_request(
|
| 218 |
method=req_plan.request.method.strip().upper(),
|
| 219 |
-
url=
|
| 220 |
-
headers=
|
| 221 |
params=req_plan.request.query_params,
|
| 222 |
data=req_plan.request.form_data if req_plan.request.form_data else None,
|
| 223 |
json=req_plan.request.body if req_plan.request.body and not req_plan.request.form_data else None,
|
|
@@ -227,8 +235,8 @@ async def execute_step(request: Request, body: schemas.ExecuteStepRequest, db: S
|
|
| 227 |
opened_files = []
|
| 228 |
try:
|
| 229 |
if req_plan.request.files:
|
| 230 |
-
|
| 231 |
-
k: v for k, v in
|
| 232 |
}
|
| 233 |
for field_name, file_name in req_plan.request.files.items():
|
| 234 |
asset = db.query(database.Asset).filter(database.Asset.name == file_name).first()
|
|
@@ -249,8 +257,8 @@ async def execute_step(request: Request, body: schemas.ExecuteStepRequest, db: S
|
|
| 249 |
if files_payload:
|
| 250 |
response = await client.request(
|
| 251 |
method=req_plan.request.method.strip().upper(),
|
| 252 |
-
url=
|
| 253 |
-
headers=
|
| 254 |
params=req_plan.request.query_params,
|
| 255 |
data=req_plan.request.form_data if req_plan.request.form_data else None,
|
| 256 |
files=files_payload,
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
import json
|
| 3 |
import os
|
| 4 |
import re
|
|
|
|
| 7 |
|
| 8 |
import httpx
|
| 9 |
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
|
|
|
|
|
| 10 |
from sqlalchemy.orm import Session
|
| 11 |
from sqlalchemy.orm.attributes import flag_modified
|
| 12 |
|
| 13 |
from backend import database, schemas
|
| 14 |
+
from backend.core.limiter import limiter
|
| 15 |
from backend.core.logging import get_logger
|
| 16 |
+
from backend.core.url_guard import resolve_safe_ip
|
| 17 |
from backend.services.api_executor import prepare_step_request
|
| 18 |
from backend.services.asset_service import AssetService, storage
|
| 19 |
from backend.services.context_manager import set_context_variables
|
|
|
|
| 22 |
|
| 23 |
router = APIRouter()
|
| 24 |
logger = get_logger(__name__)
|
|
|
|
| 25 |
|
| 26 |
PLACEHOLDER_HOSTS = {"api.example.com", "example.com", "<base_url>", "your-api.com"}
|
| 27 |
|
|
|
|
| 180 |
effective_credentials = spec.login_credentials
|
| 181 |
logger.info("Using stored credentials for spec '%s'", spec.name)
|
| 182 |
|
| 183 |
+
req_plan, executor_token_usage = await asyncio.to_thread(
|
| 184 |
+
prepare_step_request,
|
| 185 |
endpoint_spec=actual_endpoint_spec,
|
| 186 |
base_url=body.base_url,
|
| 187 |
scenario="",
|
|
|
|
| 210 |
for k in [k for k in req_plan.request.query_params if k in url_keys]:
|
| 211 |
del req_plan.request.query_params[k]
|
| 212 |
|
| 213 |
+
# Resolve IP once to prevent DNS rebinding: block private IPs, then use the
|
| 214 |
+
# resolved IP for the actual request with the original Host header set manually.
|
| 215 |
+
resolved_ip = resolve_safe_ip(url)
|
| 216 |
+
parsed_for_ip = urlparse(url)
|
| 217 |
+
original_host = parsed_for_ip.hostname
|
| 218 |
+
port = parsed_for_ip.port
|
| 219 |
+
ip_url = url.replace(original_host, resolved_ip, 1)
|
| 220 |
|
| 221 |
async with httpx.AsyncClient(timeout=120.0) as client:
|
| 222 |
+
pinned_headers = dict(req_plan.request.headers)
|
| 223 |
+
pinned_headers["Host"] = original_host + (f":{port}" if port else "")
|
| 224 |
+
|
| 225 |
http_req = client.build_request(
|
| 226 |
method=req_plan.request.method.strip().upper(),
|
| 227 |
+
url=ip_url,
|
| 228 |
+
headers=pinned_headers,
|
| 229 |
params=req_plan.request.query_params,
|
| 230 |
data=req_plan.request.form_data if req_plan.request.form_data else None,
|
| 231 |
json=req_plan.request.body if req_plan.request.body and not req_plan.request.form_data else None,
|
|
|
|
| 235 |
opened_files = []
|
| 236 |
try:
|
| 237 |
if req_plan.request.files:
|
| 238 |
+
pinned_headers = {
|
| 239 |
+
k: v for k, v in pinned_headers.items() if k.lower() != "content-type"
|
| 240 |
}
|
| 241 |
for field_name, file_name in req_plan.request.files.items():
|
| 242 |
asset = db.query(database.Asset).filter(database.Asset.name == file_name).first()
|
|
|
|
| 257 |
if files_payload:
|
| 258 |
response = await client.request(
|
| 259 |
method=req_plan.request.method.strip().upper(),
|
| 260 |
+
url=ip_url,
|
| 261 |
+
headers=pinned_headers,
|
| 262 |
params=req_plan.request.query_params,
|
| 263 |
data=req_plan.request.form_data if req_plan.request.form_data else None,
|
| 264 |
files=files_payload,
|
|
@@ -8,6 +8,7 @@ class Settings(BaseSettings):
|
|
| 8 |
UPLOAD_DIR: str = "backend/uploads"
|
| 9 |
LLM_TIMEOUT: int = 60
|
| 10 |
HTTP_TIMEOUT: int = 120
|
|
|
|
| 11 |
|
| 12 |
model_config = {"env_file": ".env", "extra": "ignore"}
|
| 13 |
|
|
|
|
| 8 |
UPLOAD_DIR: str = "backend/uploads"
|
| 9 |
LLM_TIMEOUT: int = 60
|
| 10 |
HTTP_TIMEOUT: int = 120
|
| 11 |
+
RESPONSE_TRUNCATION_LIMIT: int = 4000
|
| 12 |
|
| 13 |
model_config = {"env_file": ".env", "extra": "ignore"}
|
| 14 |
|
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from slowapi import Limiter
|
| 2 |
+
from slowapi.util import get_remote_address
|
| 3 |
+
|
| 4 |
+
limiter = Limiter(key_func=get_remote_address, default_limits=["200/minute"])
|
|
@@ -18,8 +18,13 @@ _BLOCKED_NETS = [
|
|
| 18 |
_ALLOWED_SCHEMES = {"http", "https"}
|
| 19 |
|
| 20 |
|
| 21 |
-
def
|
| 22 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
try:
|
| 24 |
parsed = urlparse(url)
|
| 25 |
except Exception:
|
|
@@ -35,7 +40,6 @@ def assert_safe_url(url: str) -> None:
|
|
| 35 |
if not hostname:
|
| 36 |
raise HTTPException(status_code=400, detail="URL has no hostname.")
|
| 37 |
|
| 38 |
-
# Resolve to IP and check against blocked ranges
|
| 39 |
try:
|
| 40 |
ip_str = socket.getaddrinfo(hostname, None)[0][4][0]
|
| 41 |
ip = ipaddress.ip_address(ip_str)
|
|
@@ -48,3 +52,10 @@ def assert_safe_url(url: str) -> None:
|
|
| 48 |
status_code=400,
|
| 49 |
detail=f"Requests to private/internal addresses are not allowed (resolved to {ip}).",
|
| 50 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
_ALLOWED_SCHEMES = {"http", "https"}
|
| 19 |
|
| 20 |
|
| 21 |
+
def resolve_safe_ip(url: str) -> str:
|
| 22 |
+
"""
|
| 23 |
+
Resolve the hostname to an IP, verify it's not private, and return the IP string.
|
| 24 |
+
Raises HTTP 400 if the URL is malformed, uses a disallowed scheme, or resolves to a
|
| 25 |
+
private/internal address. The caller should use this IP directly when making the
|
| 26 |
+
request to prevent DNS rebinding (the hostname is resolved only once here).
|
| 27 |
+
"""
|
| 28 |
try:
|
| 29 |
parsed = urlparse(url)
|
| 30 |
except Exception:
|
|
|
|
| 40 |
if not hostname:
|
| 41 |
raise HTTPException(status_code=400, detail="URL has no hostname.")
|
| 42 |
|
|
|
|
| 43 |
try:
|
| 44 |
ip_str = socket.getaddrinfo(hostname, None)[0][4][0]
|
| 45 |
ip = ipaddress.ip_address(ip_str)
|
|
|
|
| 52 |
status_code=400,
|
| 53 |
detail=f"Requests to private/internal addresses are not allowed (resolved to {ip}).",
|
| 54 |
)
|
| 55 |
+
|
| 56 |
+
return ip_str
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def assert_safe_url(url: str) -> None:
|
| 60 |
+
"""Raise HTTP 400 if the URL targets a private/internal address or disallowed scheme."""
|
| 61 |
+
resolve_safe_ip(url)
|
|
@@ -1,11 +1,15 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
import uuid
|
| 3 |
-
from sqlalchemy.orm import declarative_base, sessionmaker, relationship
|
| 4 |
from datetime import datetime
|
| 5 |
-
import os
|
| 6 |
-
from dotenv import load_dotenv
|
| 7 |
from pathlib import Path
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
# Load environment variables from .env file (handling both root and subfolder execution)
|
| 10 |
env_path = Path(__file__).resolve().parent.parent / '.env'
|
| 11 |
if not env_path.exists():
|
|
@@ -16,15 +20,13 @@ load_dotenv(dotenv_path=env_path)
|
|
| 16 |
|
| 17 |
DATABASE_URL = os.getenv("DATABASE_URL")
|
| 18 |
if not DATABASE_URL:
|
| 19 |
-
print(f"DEBUG: env_path used = {env_path}")
|
| 20 |
-
print(f"DEBUG: env_path exists? = {env_path.exists()}")
|
| 21 |
raise ValueError(f"DATABASE_URL environment variable is not set. Checked at: {env_path}")
|
| 22 |
|
| 23 |
# Fix for common PostgreSQL connection string prefixes
|
| 24 |
if DATABASE_URL.startswith("postgres://"):
|
| 25 |
DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://", 1)
|
| 26 |
|
| 27 |
-
|
| 28 |
engine = create_engine(
|
| 29 |
DATABASE_URL,
|
| 30 |
pool_pre_ping=True,
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import os
|
| 3 |
import uuid
|
|
|
|
| 4 |
from datetime import datetime
|
|
|
|
|
|
|
| 5 |
from pathlib import Path
|
| 6 |
|
| 7 |
+
from dotenv import load_dotenv
|
| 8 |
+
from sqlalchemy import Column, DateTime, ForeignKey, Integer, JSON, String, Table, Text, Unicode, create_engine
|
| 9 |
+
from sqlalchemy.orm import declarative_base, relationship, sessionmaker
|
| 10 |
+
|
| 11 |
+
logger = logging.getLogger(__name__)
|
| 12 |
+
|
| 13 |
# Load environment variables from .env file (handling both root and subfolder execution)
|
| 14 |
env_path = Path(__file__).resolve().parent.parent / '.env'
|
| 15 |
if not env_path.exists():
|
|
|
|
| 20 |
|
| 21 |
DATABASE_URL = os.getenv("DATABASE_URL")
|
| 22 |
if not DATABASE_URL:
|
|
|
|
|
|
|
| 23 |
raise ValueError(f"DATABASE_URL environment variable is not set. Checked at: {env_path}")
|
| 24 |
|
| 25 |
# Fix for common PostgreSQL connection string prefixes
|
| 26 |
if DATABASE_URL.startswith("postgres://"):
|
| 27 |
DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://", 1)
|
| 28 |
|
| 29 |
+
logger.info("Initializing DB engine: %s", DATABASE_URL.split('@')[-1])
|
| 30 |
engine = create_engine(
|
| 31 |
DATABASE_URL,
|
| 32 |
pool_pre_ping=True,
|
|
@@ -5,21 +5,18 @@ from fastapi.exceptions import RequestValidationError
|
|
| 5 |
from fastapi.middleware.cors import CORSMiddleware
|
| 6 |
from fastapi.responses import JSONResponse
|
| 7 |
from fastapi.staticfiles import StaticFiles
|
| 8 |
-
from slowapi import
|
| 9 |
from slowapi.errors import RateLimitExceeded
|
| 10 |
-
from slowapi.util import get_remote_address
|
| 11 |
|
| 12 |
from backend import database
|
| 13 |
from backend.core.config import settings
|
| 14 |
from backend.core.errors import AppError, app_error_handler
|
|
|
|
| 15 |
from backend.core.logging import get_logger
|
| 16 |
from backend.app.api.routes import specs, assets, sessions, history, chat, execution
|
| 17 |
|
| 18 |
logger = get_logger(__name__)
|
| 19 |
|
| 20 |
-
# Rate limiter — keyed by client IP
|
| 21 |
-
limiter = Limiter(key_func=get_remote_address, default_limits=["200/minute"])
|
| 22 |
-
|
| 23 |
# Ensure upload directories exist
|
| 24 |
for category in ["audio", "video", "documents", "images", "specs"]:
|
| 25 |
os.makedirs(os.path.join(settings.UPLOAD_DIR, category), exist_ok=True)
|
|
@@ -40,8 +37,8 @@ app.add_middleware(
|
|
| 40 |
CORSMiddleware,
|
| 41 |
allow_origins=settings.allowed_origins_list,
|
| 42 |
allow_credentials=True,
|
| 43 |
-
allow_methods=["
|
| 44 |
-
allow_headers=["
|
| 45 |
)
|
| 46 |
|
| 47 |
|
|
|
|
| 5 |
from fastapi.middleware.cors import CORSMiddleware
|
| 6 |
from fastapi.responses import JSONResponse
|
| 7 |
from fastapi.staticfiles import StaticFiles
|
| 8 |
+
from slowapi import _rate_limit_exceeded_handler
|
| 9 |
from slowapi.errors import RateLimitExceeded
|
|
|
|
| 10 |
|
| 11 |
from backend import database
|
| 12 |
from backend.core.config import settings
|
| 13 |
from backend.core.errors import AppError, app_error_handler
|
| 14 |
+
from backend.core.limiter import limiter
|
| 15 |
from backend.core.logging import get_logger
|
| 16 |
from backend.app.api.routes import specs, assets, sessions, history, chat, execution
|
| 17 |
|
| 18 |
logger = get_logger(__name__)
|
| 19 |
|
|
|
|
|
|
|
|
|
|
| 20 |
# Ensure upload directories exist
|
| 21 |
for category in ["audio", "video", "documents", "images", "specs"]:
|
| 22 |
os.makedirs(os.path.join(settings.UPLOAD_DIR, category), exist_ok=True)
|
|
|
|
| 37 |
CORSMiddleware,
|
| 38 |
allow_origins=settings.allowed_origins_list,
|
| 39 |
allow_credentials=True,
|
| 40 |
+
allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE"],
|
| 41 |
+
allow_headers=["Content-Type", "Authorization"],
|
| 42 |
)
|
| 43 |
|
| 44 |
|
|
@@ -3,8 +3,12 @@ from typing import Dict, Any, List
|
|
| 3 |
from langchain_core.prompts import PromptTemplate
|
| 4 |
from langchain_core.output_parsers import JsonOutputParser
|
| 5 |
from pydantic import BaseModel, Field
|
|
|
|
|
|
|
| 6 |
from backend.integrations.llm.client import deterministic as _llm
|
| 7 |
|
|
|
|
|
|
|
| 8 |
# Using Pydantic to structure LangChain output for step execution
|
| 9 |
class APIRequest(BaseModel):
|
| 10 |
method: str
|
|
@@ -85,18 +89,18 @@ RETURN JSON ONLY:
|
|
| 85 |
"""
|
| 86 |
|
| 87 |
def prepare_step_request(
|
| 88 |
-
endpoint_spec: Dict[str, Any],
|
| 89 |
-
base_url: str,
|
| 90 |
-
scenario: str,
|
| 91 |
-
step_definition: Dict[str, Any],
|
| 92 |
context_variables: Dict[str, Any],
|
| 93 |
-
user_credentials: Dict[str, Any] =
|
| 94 |
-
available_assets: Dict[str, Any] =
|
| 95 |
) -> StepExecutionPlan:
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
parser = JsonOutputParser(pydantic_object=StepExecutionPlan)
|
| 101 |
|
| 102 |
prompt = PromptTemplate(
|
|
@@ -115,6 +119,14 @@ def prepare_step_request(
|
|
| 115 |
).to_messages()
|
| 116 |
|
| 117 |
content, token_usage = _llm.invoke(msg)
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
|
| 120 |
return StepExecutionPlan(**parsed_response), token_usage
|
|
|
|
| 3 |
from langchain_core.prompts import PromptTemplate
|
| 4 |
from langchain_core.output_parsers import JsonOutputParser
|
| 5 |
from pydantic import BaseModel, Field
|
| 6 |
+
from backend.core.errors import AppError
|
| 7 |
+
from backend.core.logging import get_logger
|
| 8 |
from backend.integrations.llm.client import deterministic as _llm
|
| 9 |
|
| 10 |
+
logger = get_logger(__name__)
|
| 11 |
+
|
| 12 |
# Using Pydantic to structure LangChain output for step execution
|
| 13 |
class APIRequest(BaseModel):
|
| 14 |
method: str
|
|
|
|
| 89 |
"""
|
| 90 |
|
| 91 |
def prepare_step_request(
|
| 92 |
+
endpoint_spec: Dict[str, Any],
|
| 93 |
+
base_url: str,
|
| 94 |
+
scenario: str,
|
| 95 |
+
step_definition: Dict[str, Any],
|
| 96 |
context_variables: Dict[str, Any],
|
| 97 |
+
user_credentials: Dict[str, Any] = None,
|
| 98 |
+
available_assets: Dict[str, Any] = None,
|
| 99 |
) -> StepExecutionPlan:
|
| 100 |
+
if user_credentials is None:
|
| 101 |
+
user_credentials = {}
|
| 102 |
+
if available_assets is None:
|
| 103 |
+
available_assets = {}
|
| 104 |
parser = JsonOutputParser(pydantic_object=StepExecutionPlan)
|
| 105 |
|
| 106 |
prompt = PromptTemplate(
|
|
|
|
| 119 |
).to_messages()
|
| 120 |
|
| 121 |
content, token_usage = _llm.invoke(msg)
|
| 122 |
+
try:
|
| 123 |
+
parsed_response = parser.parse(content)
|
| 124 |
+
except Exception as e:
|
| 125 |
+
logger.warning("Step request parse failed, retrying: %s", e)
|
| 126 |
+
content, token_usage = _llm.invoke(msg)
|
| 127 |
+
try:
|
| 128 |
+
parsed_response = parser.parse(content)
|
| 129 |
+
except Exception as e2:
|
| 130 |
+
raise AppError(500, f"LLM returned unparseable output after 2 attempts: {e2}")
|
| 131 |
|
| 132 |
return StepExecutionPlan(**parsed_response), token_usage
|
|
@@ -9,9 +9,12 @@ what to do (execute a step, explain an error, modify a request, generate tests,
|
|
| 9 |
import json
|
| 10 |
import re
|
| 11 |
from typing import Any, Dict, List, Optional
|
| 12 |
-
from langchain_core.messages import
|
|
|
|
| 13 |
from backend.integrations.llm.client import chat as _llm
|
| 14 |
|
|
|
|
|
|
|
| 15 |
SYSTEM_PROMPT = """You are an AI assistant integrated into an API testing tool (similar to Postman AI).
|
| 16 |
Your job is to help the user control their API testing workflow through natural language.
|
| 17 |
|
|
@@ -156,7 +159,7 @@ def process_message(
|
|
| 156 |
return parsed
|
| 157 |
|
| 158 |
except Exception as e:
|
| 159 |
-
|
| 160 |
return {
|
| 161 |
"action": "reply",
|
| 162 |
"message": "I encountered an error processing your request. Please try again.",
|
|
|
|
| 9 |
import json
|
| 10 |
import re
|
| 11 |
from typing import Any, Dict, List, Optional
|
| 12 |
+
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
|
| 13 |
+
from backend.core.logging import get_logger
|
| 14 |
from backend.integrations.llm.client import chat as _llm
|
| 15 |
|
| 16 |
+
logger = get_logger(__name__)
|
| 17 |
+
|
| 18 |
SYSTEM_PROMPT = """You are an AI assistant integrated into an API testing tool (similar to Postman AI).
|
| 19 |
Your job is to help the user control their API testing workflow through natural language.
|
| 20 |
|
|
|
|
| 159 |
return parsed
|
| 160 |
|
| 161 |
except Exception as e:
|
| 162 |
+
logger.error("Chat AI error: %s", e)
|
| 163 |
return {
|
| 164 |
"action": "reply",
|
| 165 |
"message": "I encountered an error processing your request. Please try again.",
|
|
@@ -2,6 +2,7 @@ import json
|
|
| 2 |
from typing import Dict, Any
|
| 3 |
from langchain_core.prompts import PromptTemplate
|
| 4 |
from langchain_core.output_parsers import JsonOutputParser
|
|
|
|
| 5 |
from backend.core.logging import get_logger
|
| 6 |
from backend.schemas import AnalyzeResponse
|
| 7 |
from backend.integrations.llm.client import deterministic as _llm
|
|
@@ -47,9 +48,14 @@ def analyze_api_response(
|
|
| 47 |
input_variables=["step_info", "status_code", "response_body"],
|
| 48 |
)
|
| 49 |
|
| 50 |
-
|
| 51 |
-
if len(response_body) >
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
msg = prompt.format_prompt(
|
| 55 |
step_info=f"Step {step}: {endpoint} - {purpose}",
|
|
|
|
| 2 |
from typing import Dict, Any
|
| 3 |
from langchain_core.prompts import PromptTemplate
|
| 4 |
from langchain_core.output_parsers import JsonOutputParser
|
| 5 |
+
from backend.core.config import settings
|
| 6 |
from backend.core.logging import get_logger
|
| 7 |
from backend.schemas import AnalyzeResponse
|
| 8 |
from backend.integrations.llm.client import deterministic as _llm
|
|
|
|
| 48 |
input_variables=["step_info", "status_code", "response_body"],
|
| 49 |
)
|
| 50 |
|
| 51 |
+
limit = settings.RESPONSE_TRUNCATION_LIMIT
|
| 52 |
+
if len(response_body) > limit:
|
| 53 |
+
try:
|
| 54 |
+
parsed_body = json.loads(response_body)
|
| 55 |
+
response_body = json.dumps(parsed_body, separators=(',', ':'))[:limit]
|
| 56 |
+
except (json.JSONDecodeError, ValueError):
|
| 57 |
+
half = limit // 2
|
| 58 |
+
response_body = response_body[:half] + "\n... [TRUNCATED] ...\n" + response_body[-half:]
|
| 59 |
|
| 60 |
msg = prompt.format_prompt(
|
| 61 |
step_info=f"Step {step}: {endpoint} - {purpose}",
|
Git LFS Details
|
Git LFS Details
|
Git LFS Details
|