Amitkasar commited on
Commit
fbefffb
·
1 Parent(s): 7457a6f

Add Supabase storage and DB env handling

Browse files

Introduce Supabase storage support and improve environment loading for the database. .env.example updated with DATABASE_URL, SUPABASE_* keys and storage bucket example. backend/database.py now loads .env (handles running from repo root or subfolder), requires DATABASE_URL, normalizes postgres URL prefix, removes sqlite-specific connect_args and legacy migration code, and adds a file_path column to SpecDocument. backend/services/storage.py adds abstract method docs, ensures file pointers are reset for local saves, and implements SupabaseStorageProvider (upload/delete/get_public_url). backend/services/asset_service.py selects Supabase vs local provider based on env, logs provider choice, and updates asset upload logic to save files to storage and record file paths. backend/main.py upload_spec now saves the uploaded spec to storage, records file_path on the SpecDocument, and adds basic error logging. requirements.txt updated to relax httpx version and add supabase and websockets dependencies.

.env.example CHANGED
@@ -8,10 +8,16 @@ MISTRAL_MODEL=mistral-large-latest
8
  # On Render, the PORT will be assigned automatically.
9
  PORT=8001
10
 
11
- # Database Configuration
12
- # Local default: sqlite:///./backend/apitest.db
13
- # Render: Paste your External Database URL here
14
- # DATABASE_URL=postgresql://user:password@hostname:5432/dbname
 
 
 
 
 
 
15
 
16
  # Frontend Configuration (Vite)
17
  # Points to the Backend URL.
 
8
  # On Render, the PORT will be assigned automatically.
9
  PORT=8001
10
 
11
+ # Database Configuration (Supabase PostgreSQL)
12
+ # Get your Connection String from Supabase -> Project Settings -> Database
13
+ DATABASE_URL=postgresql://postgres.your-project:your-password@aws-0-region.pooler.supabase.com:5432/postgres
14
+
15
+ # Supabase Storage & Client Configuration
16
+ # Get these from Supabase -> Project Settings -> API
17
+ SUPABASE_URL=https://your-project.supabase.co
18
+ # Use the Service Role Key for backend storage access
19
+ SUPABASE_SERVICE_ROLE_KEY=your-service-role-key-here
20
+ SUPABASE_STORAGE_BUCKET=assets
21
 
22
  # Frontend Configuration (Vite)
23
  # Points to the Backend URL.
backend/database.py CHANGED
@@ -3,17 +3,28 @@ import uuid
3
  from sqlalchemy.orm import declarative_base, sessionmaker, relationship
4
  from datetime import datetime
5
  import os
 
 
6
 
7
- DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./backend/apitest.db")
8
- # Fix for Render/Postgres who uses 'postgres://' instead of 'postgresql://'
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  if DATABASE_URL.startswith("postgres://"):
10
  DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://", 1)
11
 
12
- connect_args = {}
13
- if DATABASE_URL.startswith("sqlite"):
14
- connect_args["check_same_thread"] = False
15
-
16
- engine = create_engine(DATABASE_URL, connect_args=connect_args)
17
  SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
18
 
19
  Base = declarative_base()
@@ -37,6 +48,7 @@ class SpecDocument(Base):
37
  login_credentials = Column(JSON, nullable=True) # Per-spec stored credentials
38
  execution_plan = Column(JSON, nullable=True) # Persisted LLM execution plan
39
  scenario = Column(Text, nullable=True) # Saved test scenario text
 
40
  uploaded_at = Column(DateTime, default=datetime.utcnow)
41
 
42
  # Relationships
@@ -95,73 +107,9 @@ class Asset(Base):
95
  spec = relationship("SpecDocument", back_populates="assets")
96
  collections = relationship("Collection", secondary=asset_collections, back_populates="assets")
97
 
98
- # Create tables
99
  Base.metadata.create_all(bind=engine)
100
 
101
- def run_migrations():
102
- """Apply any pending schema migrations safely."""
103
- with engine.connect() as conn:
104
- # Columns to add to spec_documents
105
- spec_cols = [
106
- ("execution_plan", "TEXT"),
107
- ("scenario", "TEXT")
108
- ]
109
- for col_name, col_type in spec_cols:
110
- try:
111
- conn.execute(text(f"ALTER TABLE spec_documents ADD COLUMN {col_name} {col_type}"))
112
- conn.commit()
113
- except Exception:
114
- pass
115
-
116
- # Columns to add to execution_history
117
- hist_cols = [
118
- ("analysis_json", "TEXT"),
119
- ("step_definition_json", "TEXT")
120
- ]
121
- for col_name, col_type in hist_cols:
122
- try:
123
- conn.execute(text(f"ALTER TABLE execution_history ADD COLUMN {col_name} {col_type}"))
124
- conn.commit()
125
- except Exception:
126
- pass
127
-
128
- # Columns to add to assets
129
- asset_cols = [
130
- ("description", "TEXT"),
131
- ("tags", "TEXT"), # JSON handled as TEXT in sqlite
132
- ("metadata_json", "TEXT")
133
- ]
134
- for col_name, col_type in asset_cols:
135
- try:
136
- conn.execute(text(f"ALTER TABLE assets ADD COLUMN {col_name} {col_type}"))
137
- conn.commit()
138
- except Exception:
139
- pass
140
-
141
- # Columns to add to sessions
142
- session_cols = [
143
- ("spec_id", "VARCHAR"),
144
- ("spec_name", "VARCHAR"),
145
- ("base_url", "VARCHAR"),
146
- ("scenario", "TEXT")
147
- ]
148
- for col_name, col_type in session_cols:
149
- try:
150
- conn.execute(text(f"ALTER TABLE sessions ADD COLUMN {col_name} {col_type}"))
151
- conn.commit()
152
- except Exception:
153
- pass
154
-
155
- # Ensure collection table exists
156
- try:
157
- conn.execute(text("CREATE TABLE IF NOT EXISTS collections (id VARCHAR PRIMARY KEY, name VARCHAR, description TEXT, created_at TIMESTAMP)"))
158
- conn.execute(text("CREATE TABLE IF NOT EXISTS asset_collections (asset_id VARCHAR, collection_id VARCHAR, PRIMARY KEY (asset_id, collection_id))"))
159
- conn.commit()
160
- except Exception:
161
- pass
162
-
163
- run_migrations()
164
-
165
 
166
  def get_db():
167
  db = SessionLocal()
 
3
  from sqlalchemy.orm import declarative_base, sessionmaker, relationship
4
  from datetime import datetime
5
  import os
6
+ from dotenv import load_dotenv
7
+ from pathlib import Path
8
 
9
+ # Load environment variables from .env file (handling both root and subfolder execution)
10
+ env_path = Path(__file__).resolve().parent.parent / '.env'
11
+ if not env_path.exists():
12
+ # Fallback to current directory for edge cases
13
+ env_path = Path('.env').resolve()
14
+
15
+ load_dotenv(dotenv_path=env_path)
16
+
17
+ DATABASE_URL = os.getenv("DATABASE_URL")
18
+ if not DATABASE_URL:
19
+ print(f"DEBUG: env_path used = {env_path}")
20
+ print(f"DEBUG: env_path exists? = {env_path.exists()}")
21
+ raise ValueError(f"DATABASE_URL environment variable is not set. Checked at: {env_path}")
22
+
23
+ # Fix for common PostgreSQL connection string prefixes
24
  if DATABASE_URL.startswith("postgres://"):
25
  DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://", 1)
26
 
27
+ engine = create_engine(DATABASE_URL)
 
 
 
 
28
  SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
29
 
30
  Base = declarative_base()
 
48
  login_credentials = Column(JSON, nullable=True) # Per-spec stored credentials
49
  execution_plan = Column(JSON, nullable=True) # Persisted LLM execution plan
50
  scenario = Column(Text, nullable=True) # Saved test scenario text
51
+ file_path = Column(String, nullable=True) # Reference to file in Supabase Storage
52
  uploaded_at = Column(DateTime, default=datetime.utcnow)
53
 
54
  # Relationships
 
107
  spec = relationship("SpecDocument", back_populates="assets")
108
  collections = relationship("Collection", secondary=asset_collections, back_populates="assets")
109
 
110
+ # Create tables if they don't exist
111
  Base.metadata.create_all(bind=engine)
112
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
 
114
  def get_db():
115
  db = SessionLocal()
backend/main.py CHANGED
@@ -18,6 +18,7 @@ from backend.services.postman_parser import parse_postman_collection as parse_po
18
  from backend.services.llm_planner import generate_execution_plan
19
  from backend.services.api_executor import prepare_step_request
20
  from backend.services.response_analyzer import analyze_api_response
 
21
  from backend.services.context_manager import get_context, set_context_variables
22
  from backend.services.asset_service import AssetService
23
 
@@ -64,8 +65,8 @@ def create_session(db: Session = Depends(database.get_db)):
64
 
65
  @app.post("/upload-spec")
66
  async def upload_spec(file: UploadFile = File(...), db: Session = Depends(database.get_db)):
67
- content = await file.read()
68
  try:
 
69
  decoded_content = content.decode("utf-8", errors="replace")
70
 
71
  # Detect if it's a Postman collection
@@ -92,13 +93,22 @@ async def upload_spec(file: UploadFile = File(...), db: Session = Depends(databa
92
 
93
  safe_spec = json.loads(json.dumps(parsed_spec, ensure_ascii=True))
94
 
95
- # Persist the spec to the database
96
  spec_id = str(uuid.uuid4())
 
 
 
 
 
 
 
 
 
97
  spec_doc = database.SpecDocument(
98
  id=spec_id,
99
  name=file.filename,
100
  raw_content=decoded_content,
101
- parsed_json=safe_spec
 
102
  )
103
  db.add(spec_doc)
104
  db.commit()
@@ -107,6 +117,11 @@ async def upload_spec(file: UploadFile = File(...), db: Session = Depends(databa
107
  safe_spec["id"] = spec_id
108
 
109
  return {"status": "success", "spec_id": spec_id, "parsed_spec": safe_spec}
 
 
 
 
 
110
  except Exception as e:
111
  raise HTTPException(status_code=400, detail=f"Failed to parse spec: {str(e)}")
112
 
 
18
  from backend.services.llm_planner import generate_execution_plan
19
  from backend.services.api_executor import prepare_step_request
20
  from backend.services.response_analyzer import analyze_api_response
21
+ from backend.services.asset_service import storage
22
  from backend.services.context_manager import get_context, set_context_variables
23
  from backend.services.asset_service import AssetService
24
 
 
65
 
66
  @app.post("/upload-spec")
67
  async def upload_spec(file: UploadFile = File(...), db: Session = Depends(database.get_db)):
 
68
  try:
69
+ content = await file.read()
70
  decoded_content = content.decode("utf-8", errors="replace")
71
 
72
  # Detect if it's a Postman collection
 
93
 
94
  safe_spec = json.loads(json.dumps(parsed_spec, ensure_ascii=True))
95
 
 
96
  spec_id = str(uuid.uuid4())
97
+ # PERSIST to Storage (Supabase or Local)
98
+ file_path = f"specs/{spec_id}_{file.filename}"
99
+
100
+ # Log for debugging
101
+ print(f"[DEBUG] Uploading {file.filename} to {file_path}")
102
+
103
+ storage.save(file.file, file_path)
104
+
105
+ # Persist the spec to the database
106
  spec_doc = database.SpecDocument(
107
  id=spec_id,
108
  name=file.filename,
109
  raw_content=decoded_content,
110
+ parsed_json=safe_spec,
111
+ file_path=file_path
112
  )
113
  db.add(spec_doc)
114
  db.commit()
 
117
  safe_spec["id"] = spec_id
118
 
119
  return {"status": "success", "spec_id": spec_id, "parsed_spec": safe_spec}
120
+ except Exception as e:
121
+ print(f"[ERROR] upload_spec failed: {str(e)}")
122
+ import traceback
123
+ traceback.print_exc()
124
+ raise HTTPException(status_code=400, detail=str(e))
125
  except Exception as e:
126
  raise HTTPException(status_code=400, detail=f"Failed to parse spec: {str(e)}")
127
 
backend/services/asset_service.py CHANGED
@@ -3,9 +3,27 @@ import uuid
3
  from typing import List, Optional, Dict, Any
4
  from sqlalchemy.orm import Session
5
  from ..database import Asset, Collection, SpecDocument
6
- from .storage import LocalStorageProvider
7
 
8
- storage = LocalStorageProvider()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
 
10
  def get_category(filename: str, content_type: str) -> str:
11
  ext = os.path.splitext(filename)[1].lower()
@@ -24,16 +42,30 @@ class AssetService:
24
  file: Any,
25
  spec_id: Optional[str] = None,
26
  description: Optional[str] = None,
27
- tags: List[str] = []
 
28
  ) -> Asset:
29
  file_id = str(uuid.uuid4())
30
  file_ext = os.path.splitext(file.filename)[1]
31
  category = get_category(file.filename, file.content_type)
32
 
33
- relative_path = os.path.join(category, f"{file_id}{file_ext}")
34
- full_path = storage.save(file.file, relative_path)
 
 
 
 
 
35
 
36
- file_size = os.path.getsize(full_path)
 
 
 
 
 
 
 
 
37
 
38
  new_asset = Asset(
39
  id=file_id,
 
3
  from typing import List, Optional, Dict, Any
4
  from sqlalchemy.orm import Session
5
  from ..database import Asset, Collection, SpecDocument
6
+ from .storage import LocalStorageProvider, SupabaseStorageProvider
7
 
8
+ # Load environment for storage configuration
9
+ SUPABASE_URL = os.getenv("SUPABASE_URL")
10
+ # Use Service Role Key for storage operations (bypass RLS)
11
+ # We check common names for the service role key
12
+ SUPABASE_KEY = (
13
+ os.getenv("SUPABASE_SERVICE_ROLE_KEY") or
14
+ os.getenv("SUPABASE_SECRET_KEY") or
15
+ os.getenv("SUPABASE_KEY")
16
+ )
17
+ SUPABASE_BUCKET = os.getenv("SUPABASE_STORAGE_BUCKET", "assets")
18
+
19
+ if SUPABASE_URL and SUPABASE_KEY:
20
+ storage = SupabaseStorageProvider(SUPABASE_URL, SUPABASE_KEY, SUPABASE_BUCKET)
21
+ # Check if we're using a service role key
22
+ key_type = "Service Role Key" if os.getenv("SUPABASE_SERVICE_ROLE_KEY") else "API Key"
23
+ print(f"[STORAGE] Using Supabase Storage (Bucket: {SUPABASE_BUCKET}) with {key_type}")
24
+ else:
25
+ storage = LocalStorageProvider()
26
+ print("[STORAGE] Using Local Storage")
27
 
28
  def get_category(filename: str, content_type: str) -> str:
29
  ext = os.path.splitext(filename)[1].lower()
 
42
  file: Any,
43
  spec_id: Optional[str] = None,
44
  description: Optional[str] = None,
45
+ tags: List[str] = [],
46
+ folder: Optional[str] = None
47
  ) -> Asset:
48
  file_id = str(uuid.uuid4())
49
  file_ext = os.path.splitext(file.filename)[1]
50
  category = get_category(file.filename, file.content_type)
51
 
52
+ # Use provided folder or category as prefix
53
+ prefix = folder if folder else category
54
+ relative_path = f"{prefix}/{file_id}{file_ext}"
55
+
56
+ # Save to storage (returns the path or URL)
57
+ # Note: LocalStorage returns full path, Supabase returns relative path/URL
58
+ storage.save(file.file, relative_path)
59
 
60
+ # If local, use os.path.getsize; if cloud, we might not have it easily here
61
+ # but for now we keep it simple.
62
+ file_size = 0
63
+ try:
64
+ if isinstance(storage, LocalStorageProvider):
65
+ full_path = storage.get_full_path(relative_path)
66
+ file_size = os.path.getsize(full_path)
67
+ except:
68
+ pass
69
 
70
  new_asset = Asset(
71
  id=file_id,
backend/services/storage.py CHANGED
@@ -6,15 +6,18 @@ from typing import BinaryIO
6
  class StorageProvider(ABC):
7
  @abstractmethod
8
  def save(self, file_content: BinaryIO, path: str) -> str:
9
- pass
 
10
 
11
  @abstractmethod
12
  def delete(self, path: str):
 
13
  pass
14
 
15
  @abstractmethod
16
  def get_full_path(self, path: str) -> str:
17
- pass
 
18
 
19
  class LocalStorageProvider(StorageProvider):
20
  def __init__(self, base_dir: str = "backend/uploads"):
@@ -24,6 +27,9 @@ class LocalStorageProvider(StorageProvider):
24
  def save(self, file_content: BinaryIO, relative_path: str) -> str:
25
  full_path = os.path.join(self.base_dir, relative_path)
26
  os.makedirs(os.path.dirname(full_path), exist_ok=True)
 
 
 
27
  with open(full_path, "wb") as buffer:
28
  shutil.copyfileobj(file_content, buffer)
29
  return full_path
@@ -35,3 +41,51 @@ class LocalStorageProvider(StorageProvider):
35
 
36
  def get_full_path(self, relative_path: str) -> str:
37
  return os.path.join(self.base_dir, relative_path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  class StorageProvider(ABC):
7
  @abstractmethod
8
  def save(self, file_content: BinaryIO, path: str) -> str:
9
+ """Saves the file and returns the path/URL."""
10
+ return ""
11
 
12
  @abstractmethod
13
  def delete(self, path: str):
14
+ """Deletes the file at the given path."""
15
  pass
16
 
17
  @abstractmethod
18
  def get_full_path(self, path: str) -> str:
19
+ """Returns the full accessible path or URL."""
20
+ return ""
21
 
22
  class LocalStorageProvider(StorageProvider):
23
  def __init__(self, base_dir: str = "backend/uploads"):
 
27
  def save(self, file_content: BinaryIO, relative_path: str) -> str:
28
  full_path = os.path.join(self.base_dir, relative_path)
29
  os.makedirs(os.path.dirname(full_path), exist_ok=True)
30
+ # Ensure pointer is at the start
31
+ if hasattr(file_content, 'seek'):
32
+ file_content.seek(0)
33
  with open(full_path, "wb") as buffer:
34
  shutil.copyfileobj(file_content, buffer)
35
  return full_path
 
41
 
42
  def get_full_path(self, relative_path: str) -> str:
43
  return os.path.join(self.base_dir, relative_path)
44
+
45
+ class SupabaseStorageProvider(StorageProvider):
46
+ def __init__(self, url: str, key: str, bucket: str = "assets"):
47
+ try:
48
+ from supabase import create_client, Client
49
+ self.client: Client = create_client(url, key)
50
+ self.bucket = bucket
51
+ except ImportError:
52
+ raise ImportError("Please install 'supabase' python package to use SupabaseStorageProvider")
53
+
54
+ def save(self, file_content: BinaryIO, relative_path: str) -> str:
55
+ # Supabase storage expects bytes or a file-like object
56
+ # We need to ensure the pointer is at the start if it was read before
57
+ if hasattr(file_content, 'seek'):
58
+ file_content.seek(0)
59
+
60
+ # Replace backslashes with forward slashes for cross-platform compatibility in Supabase
61
+ path = relative_path.replace("\\", "/")
62
+
63
+ # Binary content read
64
+ data = file_content.read()
65
+
66
+ # Upload to Supabase
67
+ try:
68
+ res = self.client.storage.from_(self.bucket).upload(
69
+ path=path,
70
+ file=data,
71
+ file_options={"upsert": "true"}
72
+ )
73
+ # Some versions of the client might return a dict with 'error'
74
+ if isinstance(res, dict) and res.get("error"):
75
+ raise Exception(f"Supabase upload error: {res['error']}")
76
+ except Exception as e:
77
+ print(f"[STORAGE] Supabase upload exception: {e}")
78
+ raise
79
+
80
+ # Returns the path in the bucket
81
+ return path
82
+
83
+ def delete(self, relative_path: str):
84
+ path = relative_path.replace("\\", "/")
85
+ self.client.storage.from_(self.bucket).remove([path])
86
+
87
+ def get_full_path(self, relative_path: str) -> str:
88
+ # For Supabase, we return the public URL or just the relative path
89
+ # depending on how it's used. Here we return the public URL.
90
+ res = self.client.storage.from_(self.bucket).get_public_url(relative_path.replace("\\", "/"))
91
+ return res
requirements.txt CHANGED
@@ -2,7 +2,7 @@ fastapi>=0.109.0
2
  uvicorn[standard]>=0.27.0
3
  pydantic>=2.6.0
4
  streamlit==1.28.2
5
- httpx==0.25.1
6
  python-dotenv==1.0.0
7
  mistralai==0.4.2
8
  sqlalchemy==2.0.23
@@ -11,3 +11,5 @@ langchain==0.1.0
11
  langchain-mistralai==0.0.1
12
  gunicorn==21.2.0
13
  psycopg2-binary==2.9.9
 
 
 
2
  uvicorn[standard]>=0.27.0
3
  pydantic>=2.6.0
4
  streamlit==1.28.2
5
+ httpx>=0.27.2
6
  python-dotenv==1.0.0
7
  mistralai==0.4.2
8
  sqlalchemy==2.0.23
 
11
  langchain-mistralai==0.0.1
12
  gunicorn==21.2.0
13
  psycopg2-binary==2.9.9
14
+ supabase==2.28.2
15
+ websockets==15.0.1