Spaces:
Running
Running
Phase 0a: SSRF protection + auth on docking/sequencing, worker design doc
Browse files- bioai-platform/backend/app/routers/docking.py +21 -13
- bioai-platform/backend/app/routers/sequencing.py +18 -10
- bioai-platform/backend/app/routers/structures.py +42 -0
- bioai-platform/backend/app/services/ssrf.py +92 -0
- bioai-platform/backend/migrations/004_auth_user_id.sql +10 -0
- bioai-platform/frontend/src/app/(dashboard)/analyze/structure/page.tsx +8 -3
- bioai-platform/frontend/src/components/DockingViewer.tsx +71 -2
- bioai-platform/frontend/src/lib/api.ts +11 -0
- durable-worker-design.md +267 -0
bioai-platform/backend/app/routers/docking.py
CHANGED
|
@@ -4,11 +4,13 @@ import asyncio
|
|
| 4 |
import json
|
| 5 |
import math
|
| 6 |
import re
|
| 7 |
-
from fastapi import APIRouter, HTTPException
|
| 8 |
from pydantic import BaseModel, Field
|
| 9 |
from typing import Any, Optional
|
| 10 |
|
| 11 |
from app.services.supabase import get_client
|
|
|
|
|
|
|
| 12 |
|
| 13 |
router = APIRouter(prefix="/api/docking", tags=["Docking"])
|
| 14 |
_TABLE = "docking_jobs"
|
|
@@ -91,6 +93,7 @@ def _run_docking_sync(job_id: str, payload: dict):
|
|
| 91 |
run_vina,
|
| 92 |
)
|
| 93 |
import urllib.request
|
|
|
|
| 94 |
|
| 95 |
pdb_id = payload.get("pdb_id", "").strip().upper()
|
| 96 |
pdb_url = payload.get("pdb_url", "").strip()
|
|
@@ -101,6 +104,7 @@ def _run_docking_sync(job_id: str, payload: dict):
|
|
| 101 |
# 1. Obtain PDB text
|
| 102 |
pdb_text: str | None = None
|
| 103 |
if pdb_url:
|
|
|
|
| 104 |
try:
|
| 105 |
pdb_text = urllib.request.urlopen(pdb_url, timeout=30).read().decode("utf-8", errors="replace")
|
| 106 |
except Exception:
|
|
@@ -176,7 +180,9 @@ def _run_docking_sync(job_id: str, payload: dict):
|
|
| 176 |
}
|
| 177 |
|
| 178 |
supabase.table(_TABLE).update({
|
| 179 |
-
|
|
|
|
|
|
|
| 180 |
"result_sdf": json.dumps(result_obj),
|
| 181 |
}).eq("id", job_id).execute()
|
| 182 |
|
|
@@ -639,31 +645,32 @@ def _summarize_pose_interactions(protein_pdb: str, output_pdbqt: str) -> list[di
|
|
| 639 |
# ---------------------------------------------------------------------------
|
| 640 |
|
| 641 |
@router.post("/run", response_model=DockingJobResponse)
|
| 642 |
-
async def create_docking_job(body: DockingJobCreate):
|
| 643 |
supabase = get_client()
|
| 644 |
_prune_old(supabase)
|
| 645 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 646 |
import uuid, datetime
|
| 647 |
job_id = str(uuid.uuid4())
|
| 648 |
now = datetime.datetime.utcnow().isoformat()
|
| 649 |
|
| 650 |
-
# Only INSERT columns guaranteed to exist in every Supabase docking_jobs schema.
|
| 651 |
-
# Runtime params (grid, exhaustiveness, etc.) are passed to the worker in-memory.
|
| 652 |
insert_row = {
|
| 653 |
"id": job_id,
|
| 654 |
"status": "queued",
|
| 655 |
"ligand_smiles": body.smiles,
|
|
|
|
| 656 |
}
|
| 657 |
try:
|
| 658 |
supabase.table(_TABLE).insert(insert_row).execute()
|
| 659 |
except Exception as e:
|
| 660 |
-
# If even ligand_smiles is missing, try bare minimum
|
| 661 |
if "ligand_smiles" in str(e):
|
| 662 |
-
supabase.table(_TABLE).insert({"id": job_id, "status": "queued"}).execute()
|
| 663 |
else:
|
| 664 |
raise
|
| 665 |
|
| 666 |
-
# Pass full params to the background worker (not stored in DB)
|
| 667 |
worker_payload = {
|
| 668 |
**insert_row,
|
| 669 |
"pdb_id": body.pdb_id,
|
|
@@ -680,18 +687,18 @@ async def create_docking_job(body: DockingJobCreate):
|
|
| 680 |
|
| 681 |
|
| 682 |
@router.get("/status/{job_id}", response_model=DockingJobResponse)
|
| 683 |
-
async def get_docking_job(job_id: str):
|
| 684 |
supabase = get_client()
|
| 685 |
-
result = supabase.table(_TABLE).select("*").eq("id", job_id).single().execute()
|
| 686 |
if not result.data:
|
| 687 |
raise HTTPException(status_code=404, detail="Docking job not found")
|
| 688 |
return DockingJobResponse(**_row_to_response(result.data))
|
| 689 |
|
| 690 |
|
| 691 |
@router.get("/result/{job_id}/pdb")
|
| 692 |
-
async def get_docking_pdb(job_id: str):
|
| 693 |
supabase = get_client()
|
| 694 |
-
result = supabase.table(_TABLE).select("result_sdf").eq("id", job_id).single().execute()
|
| 695 |
if not result.data or not result.data.get("result_sdf"):
|
| 696 |
raise HTTPException(status_code=404, detail="Docking result not found")
|
| 697 |
try:
|
|
@@ -706,11 +713,12 @@ async def get_docking_pdb(job_id: str):
|
|
| 706 |
|
| 707 |
|
| 708 |
@router.get("")
|
| 709 |
-
async def list_docking_jobs(limit: int = 50):
|
| 710 |
supabase = get_client()
|
| 711 |
rows = (
|
| 712 |
supabase.table(_TABLE)
|
| 713 |
.select("*")
|
|
|
|
| 714 |
.order("created_at", desc=True)
|
| 715 |
.limit(limit)
|
| 716 |
.execute()
|
|
|
|
| 4 |
import json
|
| 5 |
import math
|
| 6 |
import re
|
| 7 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 8 |
from pydantic import BaseModel, Field
|
| 9 |
from typing import Any, Optional
|
| 10 |
|
| 11 |
from app.services.supabase import get_client
|
| 12 |
+
from app.services.auth import require_user_id
|
| 13 |
+
from app.services.ssrf import validate_url
|
| 14 |
|
| 15 |
router = APIRouter(prefix="/api/docking", tags=["Docking"])
|
| 16 |
_TABLE = "docking_jobs"
|
|
|
|
| 93 |
run_vina,
|
| 94 |
)
|
| 95 |
import urllib.request
|
| 96 |
+
from app.services.ssrf import validate_url
|
| 97 |
|
| 98 |
pdb_id = payload.get("pdb_id", "").strip().upper()
|
| 99 |
pdb_url = payload.get("pdb_url", "").strip()
|
|
|
|
| 104 |
# 1. Obtain PDB text
|
| 105 |
pdb_text: str | None = None
|
| 106 |
if pdb_url:
|
| 107 |
+
validate_url(pdb_url) # SSRF guard even in worker
|
| 108 |
try:
|
| 109 |
pdb_text = urllib.request.urlopen(pdb_url, timeout=30).read().decode("utf-8", errors="replace")
|
| 110 |
except Exception:
|
|
|
|
| 180 |
}
|
| 181 |
|
| 182 |
supabase.table(_TABLE).update({
|
| 183 |
+
# Keep the persisted value aligned with the frontend's terminal
|
| 184 |
+
# status contract (and the other job runners in this service).
|
| 185 |
+
"status": "complete",
|
| 186 |
"result_sdf": json.dumps(result_obj),
|
| 187 |
}).eq("id", job_id).execute()
|
| 188 |
|
|
|
|
| 645 |
# ---------------------------------------------------------------------------
|
| 646 |
|
| 647 |
@router.post("/run", response_model=DockingJobResponse)
|
| 648 |
+
async def create_docking_job(body: DockingJobCreate, user_id: str = Depends(require_user_id)):
|
| 649 |
supabase = get_client()
|
| 650 |
_prune_old(supabase)
|
| 651 |
|
| 652 |
+
# SSRF validation on user-supplied URL
|
| 653 |
+
if body.pdb_url:
|
| 654 |
+
validate_url(body.pdb_url)
|
| 655 |
+
|
| 656 |
import uuid, datetime
|
| 657 |
job_id = str(uuid.uuid4())
|
| 658 |
now = datetime.datetime.utcnow().isoformat()
|
| 659 |
|
|
|
|
|
|
|
| 660 |
insert_row = {
|
| 661 |
"id": job_id,
|
| 662 |
"status": "queued",
|
| 663 |
"ligand_smiles": body.smiles,
|
| 664 |
+
"user_id": user_id,
|
| 665 |
}
|
| 666 |
try:
|
| 667 |
supabase.table(_TABLE).insert(insert_row).execute()
|
| 668 |
except Exception as e:
|
|
|
|
| 669 |
if "ligand_smiles" in str(e):
|
| 670 |
+
supabase.table(_TABLE).insert({"id": job_id, "status": "queued", "user_id": user_id}).execute()
|
| 671 |
else:
|
| 672 |
raise
|
| 673 |
|
|
|
|
| 674 |
worker_payload = {
|
| 675 |
**insert_row,
|
| 676 |
"pdb_id": body.pdb_id,
|
|
|
|
| 687 |
|
| 688 |
|
| 689 |
@router.get("/status/{job_id}", response_model=DockingJobResponse)
|
| 690 |
+
async def get_docking_job(job_id: str, user_id: str = Depends(require_user_id)):
|
| 691 |
supabase = get_client()
|
| 692 |
+
result = supabase.table(_TABLE).select("*").eq("id", job_id).eq("user_id", user_id).single().execute()
|
| 693 |
if not result.data:
|
| 694 |
raise HTTPException(status_code=404, detail="Docking job not found")
|
| 695 |
return DockingJobResponse(**_row_to_response(result.data))
|
| 696 |
|
| 697 |
|
| 698 |
@router.get("/result/{job_id}/pdb")
|
| 699 |
+
async def get_docking_pdb(job_id: str, user_id: str = Depends(require_user_id)):
|
| 700 |
supabase = get_client()
|
| 701 |
+
result = supabase.table(_TABLE).select("result_sdf").eq("id", job_id).eq("user_id", user_id).single().execute()
|
| 702 |
if not result.data or not result.data.get("result_sdf"):
|
| 703 |
raise HTTPException(status_code=404, detail="Docking result not found")
|
| 704 |
try:
|
|
|
|
| 713 |
|
| 714 |
|
| 715 |
@router.get("")
|
| 716 |
+
async def list_docking_jobs(limit: int = 50, user_id: str = Depends(require_user_id)):
|
| 717 |
supabase = get_client()
|
| 718 |
rows = (
|
| 719 |
supabase.table(_TABLE)
|
| 720 |
.select("*")
|
| 721 |
+
.eq("user_id", user_id)
|
| 722 |
.order("created_at", desc=True)
|
| 723 |
.limit(limit)
|
| 724 |
.execute()
|
bioai-platform/backend/app/routers/sequencing.py
CHANGED
|
@@ -6,11 +6,13 @@ import uuid
|
|
| 6 |
from datetime import datetime, timezone, timedelta
|
| 7 |
from typing import Optional
|
| 8 |
|
| 9 |
-
from fastapi import APIRouter, BackgroundTasks, HTTPException
|
| 10 |
from pydantic import BaseModel
|
| 11 |
|
| 12 |
from app.deps import limiter
|
| 13 |
from app.services.supabase import get_supabase
|
|
|
|
|
|
|
| 14 |
|
| 15 |
logger = logging.getLogger(__name__)
|
| 16 |
router = APIRouter(prefix="/api/sequencing", tags=["sequencing"])
|
|
@@ -56,7 +58,7 @@ class SequencingJob(BaseModel):
|
|
| 56 |
done_at: Optional[str] = None
|
| 57 |
|
| 58 |
|
| 59 |
-
def _init(job_id: str, req: SequencingRequest) -> None:
|
| 60 |
try:
|
| 61 |
_prune_jobs()
|
| 62 |
except Exception:
|
|
@@ -66,6 +68,7 @@ def _init(job_id: str, req: SequencingRequest) -> None:
|
|
| 66 |
"fastq_url": req.fastq_url,
|
| 67 |
"reference": req.reference,
|
| 68 |
"status": "queued",
|
|
|
|
| 69 |
"result": None,
|
| 70 |
"error": None,
|
| 71 |
"done_at": None,
|
|
@@ -76,8 +79,11 @@ def _patch(job_id: str, **kw) -> None:
|
|
| 76 |
get_supabase().table(_TABLE).update(kw).eq("id", job_id).execute()
|
| 77 |
|
| 78 |
|
| 79 |
-
def _read(job_id: str) -> dict | None:
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
| 81 |
return dict(rows[0]) if rows else None
|
| 82 |
|
| 83 |
|
|
@@ -112,22 +118,24 @@ VALID_DEMO = {"synthetic", "demo", "test"}
|
|
| 112 |
|
| 113 |
|
| 114 |
@router.post("/run")
|
| 115 |
-
async def run_sequencing(req: SequencingRequest, background_tasks: BackgroundTasks):
|
| 116 |
if not req.fastq_url.strip():
|
| 117 |
raise HTTPException(400, detail="fastq_url is required")
|
| 118 |
-
if
|
| 119 |
-
|
|
|
|
|
|
|
| 120 |
|
| 121 |
job_id = str(uuid.uuid4())
|
| 122 |
-
_init(job_id, req)
|
| 123 |
background_tasks.add_task(_worker, job_id)
|
| 124 |
return {"job_id": job_id, "status": "queued"}
|
| 125 |
|
| 126 |
|
| 127 |
@router.get("/status/{job_id}")
|
| 128 |
@limiter.exempt
|
| 129 |
-
async def get_status(job_id: str):
|
| 130 |
-
job = _read(job_id)
|
| 131 |
if not job:
|
| 132 |
raise HTTPException(404, detail=f"Job {job_id} not found")
|
| 133 |
return job
|
|
|
|
| 6 |
from datetime import datetime, timezone, timedelta
|
| 7 |
from typing import Optional
|
| 8 |
|
| 9 |
+
from fastapi import APIRouter, BackgroundTasks, HTTPException, Depends
|
| 10 |
from pydantic import BaseModel
|
| 11 |
|
| 12 |
from app.deps import limiter
|
| 13 |
from app.services.supabase import get_supabase
|
| 14 |
+
from app.services.auth import require_user_id
|
| 15 |
+
from app.services.ssrf import validate_url
|
| 16 |
|
| 17 |
logger = logging.getLogger(__name__)
|
| 18 |
router = APIRouter(prefix="/api/sequencing", tags=["sequencing"])
|
|
|
|
| 58 |
done_at: Optional[str] = None
|
| 59 |
|
| 60 |
|
| 61 |
+
def _init(job_id: str, req: SequencingRequest, user_id: str) -> None:
|
| 62 |
try:
|
| 63 |
_prune_jobs()
|
| 64 |
except Exception:
|
|
|
|
| 68 |
"fastq_url": req.fastq_url,
|
| 69 |
"reference": req.reference,
|
| 70 |
"status": "queued",
|
| 71 |
+
"user_id": user_id,
|
| 72 |
"result": None,
|
| 73 |
"error": None,
|
| 74 |
"done_at": None,
|
|
|
|
| 79 |
get_supabase().table(_TABLE).update(kw).eq("id", job_id).execute()
|
| 80 |
|
| 81 |
|
| 82 |
+
def _read(job_id: str, user_id: str | None = None) -> dict | None:
|
| 83 |
+
query = get_supabase().table(_TABLE).select("*").eq("id", job_id)
|
| 84 |
+
if user_id:
|
| 85 |
+
query = query.eq("user_id", user_id)
|
| 86 |
+
rows = query.execute().data
|
| 87 |
return dict(rows[0]) if rows else None
|
| 88 |
|
| 89 |
|
|
|
|
| 118 |
|
| 119 |
|
| 120 |
@router.post("/run")
|
| 121 |
+
async def run_sequencing(req: SequencingRequest, background_tasks: BackgroundTasks, user_id: str = Depends(require_user_id)):
|
| 122 |
if not req.fastq_url.strip():
|
| 123 |
raise HTTPException(400, detail="fastq_url is required")
|
| 124 |
+
if req.fastq_url.lower() not in VALID_DEMO:
|
| 125 |
+
if not req.fastq_url.startswith(("http://", "https://")):
|
| 126 |
+
raise HTTPException(400, detail="fastq_url must be a valid URL or 'synthetic' for demo data")
|
| 127 |
+
validate_url(req.fastq_url)
|
| 128 |
|
| 129 |
job_id = str(uuid.uuid4())
|
| 130 |
+
_init(job_id, req, user_id)
|
| 131 |
background_tasks.add_task(_worker, job_id)
|
| 132 |
return {"job_id": job_id, "status": "queued"}
|
| 133 |
|
| 134 |
|
| 135 |
@router.get("/status/{job_id}")
|
| 136 |
@limiter.exempt
|
| 137 |
+
async def get_status(job_id: str, user_id: str = Depends(require_user_id)):
|
| 138 |
+
job = _read(job_id, user_id)
|
| 139 |
if not job:
|
| 140 |
raise HTTPException(404, detail=f"Job {job_id} not found")
|
| 141 |
return job
|
bioai-platform/backend/app/routers/structures.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import httpx
|
| 2 |
import re
|
|
|
|
| 3 |
from fastapi import APIRouter, HTTPException
|
| 4 |
from pydantic import BaseModel, Field
|
| 5 |
|
|
@@ -16,6 +17,10 @@ class StructureSearchRequest(BaseModel):
|
|
| 16 |
query: str = Field(..., min_length=1, description="PDB ID, UniProt accession, or keyword")
|
| 17 |
|
| 18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
def _is_pdb_id(q: str) -> bool:
|
| 20 |
return bool(re.fullmatch(r'[A-Za-z0-9]{4}', q))
|
| 21 |
|
|
@@ -150,3 +155,40 @@ async def search_pdb(req: StructureSearchRequest):
|
|
| 150 |
"score": hit.get("score", 0),
|
| 151 |
})
|
| 152 |
return {"results": results, "count": len(results)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import httpx
|
| 2 |
import re
|
| 3 |
+
from collections import defaultdict
|
| 4 |
from fastapi import APIRouter, HTTPException
|
| 5 |
from pydantic import BaseModel, Field
|
| 6 |
|
|
|
|
| 17 |
query: str = Field(..., min_length=1, description="PDB ID, UniProt accession, or keyword")
|
| 18 |
|
| 19 |
|
| 20 |
+
class StructureInventoryRequest(BaseModel):
|
| 21 |
+
pdb_id: str = Field(..., pattern=r"^[A-Za-z0-9]{4}$", description="Four-character PDB identifier")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
def _is_pdb_id(q: str) -> bool:
|
| 25 |
return bool(re.fullmatch(r'[A-Za-z0-9]{4}', q))
|
| 26 |
|
|
|
|
| 155 |
"score": hit.get("score", 0),
|
| 156 |
})
|
| 157 |
return {"results": results, "count": len(results)}
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
@router.post("/inventory")
|
| 161 |
+
async def structure_inventory(req: StructureInventoryRequest):
|
| 162 |
+
"""Return lightweight chain and non-polymer inventory for workbench controls."""
|
| 163 |
+
pdb_id = req.pdb_id.upper()
|
| 164 |
+
async with httpx.AsyncClient(timeout=20) as client:
|
| 165 |
+
response = await client.get(f"https://files.rcsb.org/download/{pdb_id}.pdb")
|
| 166 |
+
if response.status_code != 200:
|
| 167 |
+
raise HTTPException(status_code=404, detail=f"PDB file not found: {pdb_id}")
|
| 168 |
+
|
| 169 |
+
chains: dict[str, set[tuple[str, str]]] = defaultdict(set)
|
| 170 |
+
ligands: dict[tuple[str, str], dict] = {}
|
| 171 |
+
for line in response.text.splitlines():
|
| 172 |
+
if len(line) < 27:
|
| 173 |
+
continue
|
| 174 |
+
record = line[:6].strip()
|
| 175 |
+
if record not in {"ATOM", "HETATM"}:
|
| 176 |
+
continue
|
| 177 |
+
residue = line[17:20].strip() or "UNK"
|
| 178 |
+
chain = line[21].strip() or "_"
|
| 179 |
+
residue_id = f"{line[22:26].strip()}{line[26].strip()}"
|
| 180 |
+
if record == "ATOM":
|
| 181 |
+
chains[chain].add((residue, residue_id))
|
| 182 |
+
elif residue not in {"HOH", "WAT", "DOD"}:
|
| 183 |
+
key = (residue, chain)
|
| 184 |
+
ligands.setdefault(key, {"id": residue, "chain": chain, "residue_count": 0})
|
| 185 |
+
ligands[key]["residue_count"] += 1
|
| 186 |
+
|
| 187 |
+
return {
|
| 188 |
+
"pdb_id": pdb_id,
|
| 189 |
+
"chains": [
|
| 190 |
+
{"id": chain, "residue_count": len(residues)}
|
| 191 |
+
for chain, residues in sorted(chains.items())
|
| 192 |
+
],
|
| 193 |
+
"ligands": sorted(ligands.values(), key=lambda ligand: (ligand["id"], ligand["chain"])),
|
| 194 |
+
}
|
bioai-platform/backend/app/services/ssrf.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SSRF protection utilities.
|
| 2 |
+
|
| 3 |
+
Validates that user-supplied URLs point to allowed public hosts and
|
| 4 |
+
blocks requests to private/reserved IP ranges.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import ipaddress
|
| 10 |
+
import logging
|
| 11 |
+
from urllib.parse import urlparse
|
| 12 |
+
|
| 13 |
+
from fastapi import HTTPException
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
ALLOWED_HOSTS: set[str] = {
|
| 18 |
+
"files.rcsb.org",
|
| 19 |
+
"data.rcsb.org",
|
| 20 |
+
"search.rcsb.org",
|
| 21 |
+
"www.rcsb.org",
|
| 22 |
+
"alphafold.ebi.ac.uk",
|
| 23 |
+
"rest.uniprot.org",
|
| 24 |
+
"www.uniprot.org",
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
PRIVATE_NETWORKS = [
|
| 28 |
+
ipaddress.ip_network("127.0.0.0/8"),
|
| 29 |
+
ipaddress.ip_network("10.0.0.0/8"),
|
| 30 |
+
ipaddress.ip_network("172.16.0.0/12"),
|
| 31 |
+
ipaddress.ip_network("192.168.0.0/16"),
|
| 32 |
+
ipaddress.ip_network("169.254.0.0/16"),
|
| 33 |
+
ipaddress.ip_network("::1/128"),
|
| 34 |
+
ipaddress.ip_network("fc00::/7"),
|
| 35 |
+
ipaddress.ip_network("fe80::/10"),
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def validate_url(url: str, param_name: str = "url") -> None:
|
| 40 |
+
"""Validate a user-supplied URL against SSRF protections.
|
| 41 |
+
|
| 42 |
+
Checks:
|
| 43 |
+
1. URL is well-formed
|
| 44 |
+
2. Host is in the allowlist
|
| 45 |
+
3. Resolved IP is not in a private/reserved range
|
| 46 |
+
|
| 47 |
+
Raises HTTPException(400) on violation.
|
| 48 |
+
"""
|
| 49 |
+
if not url or not url.strip():
|
| 50 |
+
return
|
| 51 |
+
|
| 52 |
+
parsed = urlparse(url)
|
| 53 |
+
if parsed.scheme not in ("http", "https"):
|
| 54 |
+
raise HTTPException(status_code=400, detail=f"{param_name}: only http/https URLs are allowed")
|
| 55 |
+
|
| 56 |
+
hostname = parsed.hostname
|
| 57 |
+
if not hostname:
|
| 58 |
+
raise HTTPException(status_code=400, detail=f"{param_name}: invalid URL β no hostname")
|
| 59 |
+
|
| 60 |
+
# Check allowlist (suffix match to allow subdomains)
|
| 61 |
+
host_allowed = any(
|
| 62 |
+
hostname == allowed or hostname.endswith("." + allowed)
|
| 63 |
+
for allowed in ALLOWED_HOSTS
|
| 64 |
+
)
|
| 65 |
+
if not host_allowed:
|
| 66 |
+
raise HTTPException(
|
| 67 |
+
status_code=400,
|
| 68 |
+
detail=f"{param_name}: host '{hostname}' is not in the allowed list. "
|
| 69 |
+
f"Allowed: {', '.join(sorted(ALLOWED_HOSTS))}",
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
# Resolve IP and check for private ranges
|
| 73 |
+
try:
|
| 74 |
+
import socket
|
| 75 |
+
addrinfos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
| 76 |
+
for family, _, _, _, sockaddr in addrinfos:
|
| 77 |
+
ip = ipaddress.ip_address(sockaddr[0])
|
| 78 |
+
for net in PRIVATE_NETWORKS:
|
| 79 |
+
if ip in net:
|
| 80 |
+
raise HTTPException(
|
| 81 |
+
status_code=400,
|
| 82 |
+
detail=f"{param_name}: resolved to private IP {ip} β request blocked",
|
| 83 |
+
)
|
| 84 |
+
except HTTPException:
|
| 85 |
+
raise
|
| 86 |
+
except Exception as e:
|
| 87 |
+
logger.warning(f"SSRF DNS check failed for {hostname}: {e}")
|
| 88 |
+
# If DNS resolution fails, block the request rather than allowing it through
|
| 89 |
+
raise HTTPException(
|
| 90 |
+
status_code=400,
|
| 91 |
+
detail=f"{param_name}: could not resolve hostname β request blocked",
|
| 92 |
+
)
|
bioai-platform/backend/migrations/004_auth_user_id.sql
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- Phase 0a: Add user_id to docking_jobs and sequencing_jobs for ownership enforcement.
|
| 2 |
+
|
| 3 |
+
ALTER TABLE docking_jobs
|
| 4 |
+
ADD COLUMN IF NOT EXISTS user_id UUID REFERENCES profiles(id) ON DELETE CASCADE;
|
| 5 |
+
|
| 6 |
+
ALTER TABLE sequencing_jobs
|
| 7 |
+
ADD COLUMN IF NOT EXISTS user_id UUID REFERENCES profiles(id) ON DELETE CASCADE;
|
| 8 |
+
|
| 9 |
+
CREATE INDEX IF NOT EXISTS idx_docking_jobs_user ON docking_jobs(user_id);
|
| 10 |
+
CREATE INDEX IF NOT EXISTS idx_sequencing_jobs_user ON sequencing_jobs(user_id);
|
bioai-platform/frontend/src/app/(dashboard)/analyze/structure/page.tsx
CHANGED
|
@@ -5,11 +5,11 @@ import { useRouter } from 'next/navigation';
|
|
| 5 |
import { motion } from 'framer-motion';
|
| 6 |
import { ArrowLeft, LoaderCircle, ExternalLink, Dna } from 'lucide-react';
|
| 7 |
import { fadeUp } from '@/lib/animations';
|
| 8 |
-
import { fetchStructure } from '@/lib/api';
|
| 9 |
import { extractErrorMessage } from '@/lib/errors';
|
| 10 |
import { useAuditTrail } from '@/hooks/useAuditTrail';
|
| 11 |
import type { StructureResult } from '@/lib/api';
|
| 12 |
-
import
|
| 13 |
|
| 14 |
export default function StructurePage() {
|
| 15 |
const router = useRouter();
|
|
@@ -17,6 +17,7 @@ export default function StructurePage() {
|
|
| 17 |
const [result, setResult] = useState<StructureResult | null>(null);
|
| 18 |
const [loading, setLoading] = useState(false);
|
| 19 |
const [error, setError] = useState<string | null>(null);
|
|
|
|
| 20 |
const audit = useAuditTrail();
|
| 21 |
|
| 22 |
const handleSearch = async () => {
|
|
@@ -26,9 +27,13 @@ export default function StructurePage() {
|
|
| 26 |
setLoading(true);
|
| 27 |
setError(null);
|
| 28 |
setResult(null);
|
|
|
|
| 29 |
try {
|
| 30 |
const res = await fetchStructure(query.trim());
|
| 31 |
setResult(res);
|
|
|
|
|
|
|
|
|
|
| 32 |
audit.emitSuccess('structure_fetch', 'AlphaFold/PDB', inputSummary, `pdb:${res?.pdb_id ?? ''}`);
|
| 33 |
} catch (err: unknown) {
|
| 34 |
const errMsg = extractErrorMessage(err, 'Structure not found');
|
|
@@ -96,7 +101,7 @@ export default function StructurePage() {
|
|
| 96 |
)}
|
| 97 |
</div>
|
| 98 |
{pdbId ? (
|
| 99 |
-
<
|
| 100 |
) : (
|
| 101 |
<div className="w-full h-96 rounded-xl bg-surface-0 flex items-center justify-center">
|
| 102 |
<p className="text-sm text-text-muted">3D view not available</p>
|
|
|
|
| 5 |
import { motion } from 'framer-motion';
|
| 6 |
import { ArrowLeft, LoaderCircle, ExternalLink, Dna } from 'lucide-react';
|
| 7 |
import { fadeUp } from '@/lib/animations';
|
| 8 |
+
import { fetchStructure, getStructureInventory } from '@/lib/api';
|
| 9 |
import { extractErrorMessage } from '@/lib/errors';
|
| 10 |
import { useAuditTrail } from '@/hooks/useAuditTrail';
|
| 11 |
import type { StructureResult } from '@/lib/api';
|
| 12 |
+
import { DockingViewer } from '@/components/DockingViewer';
|
| 13 |
|
| 14 |
export default function StructurePage() {
|
| 15 |
const router = useRouter();
|
|
|
|
| 17 |
const [result, setResult] = useState<StructureResult | null>(null);
|
| 18 |
const [loading, setLoading] = useState(false);
|
| 19 |
const [error, setError] = useState<string | null>(null);
|
| 20 |
+
const [inventory, setInventory] = useState<{ chains: Array<{ id: string; residue_count: number }>; ligands: Array<{ id: string; chain: string; residue_count: number }> } | null>(null);
|
| 21 |
const audit = useAuditTrail();
|
| 22 |
|
| 23 |
const handleSearch = async () => {
|
|
|
|
| 27 |
setLoading(true);
|
| 28 |
setError(null);
|
| 29 |
setResult(null);
|
| 30 |
+
setInventory(null);
|
| 31 |
try {
|
| 32 |
const res = await fetchStructure(query.trim());
|
| 33 |
setResult(res);
|
| 34 |
+
if (res.pdb_id) {
|
| 35 |
+
getStructureInventory(res.pdb_id).then(setInventory).catch(() => setInventory(null));
|
| 36 |
+
}
|
| 37 |
audit.emitSuccess('structure_fetch', 'AlphaFold/PDB', inputSummary, `pdb:${res?.pdb_id ?? ''}`);
|
| 38 |
} catch (err: unknown) {
|
| 39 |
const errMsg = extractErrorMessage(err, 'Structure not found');
|
|
|
|
| 101 |
)}
|
| 102 |
</div>
|
| 103 |
{pdbId ? (
|
| 104 |
+
<DockingViewer pdbId={pdbId} ligandPdb="" chains={inventory?.chains} ligands={inventory?.ligands} />
|
| 105 |
) : (
|
| 106 |
<div className="w-full h-96 rounded-xl bg-surface-0 flex items-center justify-center">
|
| 107 |
<p className="text-sm text-text-muted">3D view not available</p>
|
bioai-platform/frontend/src/components/DockingViewer.tsx
CHANGED
|
@@ -13,6 +13,8 @@ interface DockingViewerProps {
|
|
| 13 |
interactions?: DockingInteraction;
|
| 14 |
height?: number | string;
|
| 15 |
backgroundColor?: string;
|
|
|
|
|
|
|
| 16 |
}
|
| 17 |
|
| 18 |
type RepresentationType = 'cartoon' | 'ball-and-stick' | 'spacefill' | 'gaussian-surface' | 'molecular-surface' | 'putty' | 'ribbon';
|
|
@@ -81,6 +83,8 @@ export function DockingViewer({
|
|
| 81 |
interactions,
|
| 82 |
height = 480,
|
| 83 |
backgroundColor = '#0d1117',
|
|
|
|
|
|
|
| 84 |
}: DockingViewerProps) {
|
| 85 |
const containerRef = useRef<HTMLDivElement>(null);
|
| 86 |
const viewerRef = useRef<PDBeElement | null>(null);
|
|
@@ -90,6 +94,7 @@ export function DockingViewer({
|
|
| 90 |
const [representation, setRepresentation] = useState<RepresentationType>('cartoon');
|
| 91 |
const [colorScheme, setColorScheme] = useState<ColorScheme>('spectrum');
|
| 92 |
const [spinning, setSpinning] = useState(false);
|
|
|
|
| 93 |
const [visibleInteractions, setVisibleInteractions] = useState<Record<InteractionLayer, boolean>>({
|
| 94 |
hbonds: true,
|
| 95 |
hydrophobic: true,
|
|
@@ -202,6 +207,43 @@ export function DockingViewer({
|
|
| 202 |
}
|
| 203 |
}, []);
|
| 204 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
const handleScreenshot = useCallback(() => {
|
| 206 |
const canvas = viewerRef.current?.querySelector('canvas');
|
| 207 |
if (!canvas) return;
|
|
@@ -224,9 +266,9 @@ export function DockingViewer({
|
|
| 224 |
|
| 225 |
// Initialize viewer ONCE per pdbId
|
| 226 |
useEffect(() => {
|
| 227 |
-
if (!pdbId
|
| 228 |
setStatus('error');
|
| 229 |
-
setError('Missing structure
|
| 230 |
return;
|
| 231 |
}
|
| 232 |
|
|
@@ -350,6 +392,25 @@ export function DockingViewer({
|
|
| 350 |
|
| 351 |
<div className="w-px h-5 bg-white/10" />
|
| 352 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
{/* Color scheme */}
|
| 354 |
<div className="relative">
|
| 355 |
<select
|
|
@@ -439,6 +500,14 @@ export function DockingViewer({
|
|
| 439 |
</div>
|
| 440 |
)}
|
| 441 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 442 |
{/* Canvas */}
|
| 443 |
<div ref={containerRef} style={{ height, minHeight: height, position: 'relative' }} />
|
| 444 |
{status === 'loading' && (
|
|
|
|
| 13 |
interactions?: DockingInteraction;
|
| 14 |
height?: number | string;
|
| 15 |
backgroundColor?: string;
|
| 16 |
+
chains?: Array<{ id: string; residue_count: number }>;
|
| 17 |
+
ligands?: Array<{ id: string; chain: string; residue_count: number }>;
|
| 18 |
}
|
| 19 |
|
| 20 |
type RepresentationType = 'cartoon' | 'ball-and-stick' | 'spacefill' | 'gaussian-surface' | 'molecular-surface' | 'putty' | 'ribbon';
|
|
|
|
| 83 |
interactions,
|
| 84 |
height = 480,
|
| 85 |
backgroundColor = '#0d1117',
|
| 86 |
+
chains = [],
|
| 87 |
+
ligands = [],
|
| 88 |
}: DockingViewerProps) {
|
| 89 |
const containerRef = useRef<HTMLDivElement>(null);
|
| 90 |
const viewerRef = useRef<PDBeElement | null>(null);
|
|
|
|
| 94 |
const [representation, setRepresentation] = useState<RepresentationType>('cartoon');
|
| 95 |
const [colorScheme, setColorScheme] = useState<ColorScheme>('spectrum');
|
| 96 |
const [spinning, setSpinning] = useState(false);
|
| 97 |
+
const [selectedChain, setSelectedChain] = useState('');
|
| 98 |
const [visibleInteractions, setVisibleInteractions] = useState<Record<InteractionLayer, boolean>>({
|
| 99 |
hbonds: true,
|
| 100 |
hydrophobic: true,
|
|
|
|
| 207 |
}
|
| 208 |
}, []);
|
| 209 |
|
| 210 |
+
const selectChain = useCallback(async (chain: string) => {
|
| 211 |
+
const viewer = viewerRef.current?.viewerInstance;
|
| 212 |
+
if (!viewer) return;
|
| 213 |
+
try {
|
| 214 |
+
if (!chain) {
|
| 215 |
+
await viewer.visual.reset({ camera: false, theme: false });
|
| 216 |
+
return;
|
| 217 |
+
}
|
| 218 |
+
await viewer.visual.select({
|
| 219 |
+
data: [{ auth_asym_id: chain, color: '#5eead4' }],
|
| 220 |
+
nonSelectedColor: '#334155',
|
| 221 |
+
keepRepresentations: true,
|
| 222 |
+
});
|
| 223 |
+
await viewer.visual.focus([{ auth_asym_id: chain }]);
|
| 224 |
+
} catch {
|
| 225 |
+
// The PDBe API can differ between deposited structures; preserve the canvas if a selection is unsupported.
|
| 226 |
+
}
|
| 227 |
+
}, []);
|
| 228 |
+
|
| 229 |
+
const focusContacts = useCallback(async () => {
|
| 230 |
+
const viewer = viewerRef.current?.viewerInstance;
|
| 231 |
+
if (!viewer || !interactions) return;
|
| 232 |
+
const residues = [
|
| 233 |
+
...interactions.hbonds,
|
| 234 |
+
...interactions.hydrophobic,
|
| 235 |
+
...interactions.pi_stacking,
|
| 236 |
+
...interactions.salt_bridges,
|
| 237 |
+
].map(contact => ({ auth_asym_id: contact.protein_chain, auth_seq_id: contact.protein_residue_seq }));
|
| 238 |
+
if (!residues.length) return;
|
| 239 |
+
try {
|
| 240 |
+
await viewer.visual.select({ data: residues, keepRepresentations: true });
|
| 241 |
+
await viewer.visual.focus(residues);
|
| 242 |
+
} catch {
|
| 243 |
+
// Best-effort viewer enhancement; the tabular fingerprint remains available.
|
| 244 |
+
}
|
| 245 |
+
}, [interactions]);
|
| 246 |
+
|
| 247 |
const handleScreenshot = useCallback(() => {
|
| 248 |
const canvas = viewerRef.current?.querySelector('canvas');
|
| 249 |
if (!canvas) return;
|
|
|
|
| 266 |
|
| 267 |
// Initialize viewer ONCE per pdbId
|
| 268 |
useEffect(() => {
|
| 269 |
+
if (!pdbId) {
|
| 270 |
setStatus('error');
|
| 271 |
+
setError('Missing structure identifier');
|
| 272 |
return;
|
| 273 |
}
|
| 274 |
|
|
|
|
| 392 |
|
| 393 |
<div className="w-px h-5 bg-white/10" />
|
| 394 |
|
| 395 |
+
{chains.length > 0 && (
|
| 396 |
+
<select
|
| 397 |
+
value={selectedChain}
|
| 398 |
+
onChange={(event) => { const chain = event.target.value; setSelectedChain(chain); selectChain(chain); }}
|
| 399 |
+
disabled={status !== 'ready'}
|
| 400 |
+
title="Focus protein chain"
|
| 401 |
+
className="rounded border border-white/10 bg-black/30 px-2 py-1 text-[10px] text-white/80 disabled:opacity-40"
|
| 402 |
+
>
|
| 403 |
+
<option value="">All chains</option>
|
| 404 |
+
{chains.map(chain => <option key={chain.id} value={chain.id}>Chain {chain.id} Β· {chain.residue_count} residues</option>)}
|
| 405 |
+
</select>
|
| 406 |
+
)}
|
| 407 |
+
|
| 408 |
+
{interactions && (
|
| 409 |
+
<button type="button" onClick={focusContacts} disabled={status !== 'ready'} className="rounded border border-white/10 px-2 py-1 text-[10px] text-white/60 hover:bg-white/5 hover:text-white disabled:opacity-40">
|
| 410 |
+
Focus pocket
|
| 411 |
+
</button>
|
| 412 |
+
)}
|
| 413 |
+
|
| 414 |
{/* Color scheme */}
|
| 415 |
<div className="relative">
|
| 416 |
<select
|
|
|
|
| 500 |
</div>
|
| 501 |
)}
|
| 502 |
|
| 503 |
+
{(chains.length > 0 || ligands.length > 0) && (
|
| 504 |
+
<div className="flex flex-wrap gap-x-4 gap-y-1 border-b border-white/10 px-3 py-2 text-[10px] text-white/50">
|
| 505 |
+
{chains.length > 0 && <span>{chains.length} protein chain{chains.length === 1 ? '' : 's'}</span>}
|
| 506 |
+
{ligands.length > 0 && <span>Deposited ligands: {ligands.map(ligand => `${ligand.id}:${ligand.chain}`).join(', ')}</span>}
|
| 507 |
+
{ligandPdb && <span className="text-accent-cyan">Docked ligand loaded</span>}
|
| 508 |
+
</div>
|
| 509 |
+
)}
|
| 510 |
+
|
| 511 |
{/* Canvas */}
|
| 512 |
<div ref={containerRef} style={{ height, minHeight: height, position: 'relative' }} />
|
| 513 |
{status === 'loading' && (
|
bioai-platform/frontend/src/lib/api.ts
CHANGED
|
@@ -175,6 +175,17 @@ export async function fetchStructure(query: string): Promise<StructureResult> {
|
|
| 175 |
return res.data;
|
| 176 |
}
|
| 177 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
/** @deprecated Unused β search is handled by fetchStructure */
|
| 179 |
export async function searchStructures(query: string): Promise<{ results: { pdb_id: string; score: number }[]; count: number }> {
|
| 180 |
const res = await api.post('/api/structures/search', { query });
|
|
|
|
| 175 |
return res.data;
|
| 176 |
}
|
| 177 |
|
| 178 |
+
export type StructureInventory = {
|
| 179 |
+
pdb_id: string;
|
| 180 |
+
chains: Array<{ id: string; residue_count: number }>;
|
| 181 |
+
ligands: Array<{ id: string; chain: string; residue_count: number }>;
|
| 182 |
+
};
|
| 183 |
+
|
| 184 |
+
export async function getStructureInventory(pdbId: string): Promise<StructureInventory> {
|
| 185 |
+
const res = await api.post('/api/structures/inventory', { pdb_id: pdbId });
|
| 186 |
+
return res.data;
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
/** @deprecated Unused β search is handled by fetchStructure */
|
| 190 |
export async function searchStructures(query: string): Promise<{ results: { pdb_id: string; score: number }[]; count: number }> {
|
| 191 |
const res = await api.post('/api/structures/search', { query });
|
durable-worker-design.md
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Durable Job Worker β Design Doc
|
| 2 |
+
|
| 3 |
+
**Scope:** replace `run_in_executor`-based docking (and sequencing/pipeline_v2) execution with a durable, restart-safe worker.
|
| 4 |
+
**Status:** proposal, not yet implemented.
|
| 5 |
+
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
## 1. Problem statement
|
| 9 |
+
|
| 10 |
+
Today, `create_docking_job` does:
|
| 11 |
+
|
| 12 |
+
```python
|
| 13 |
+
loop = asyncio.get_event_loop()
|
| 14 |
+
asyncio.ensure_future(loop.run_in_executor(None, _run_docking_sync, job_id, row))
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
This ties a 1β5 minute compute job to the lifetime of the web request's event loop / process. Consequences:
|
| 18 |
+
|
| 19 |
+
- **Any deploy, restart, or crash silently strands jobs** in `queued`/`running` forever β no process is left to finish or retry them.
|
| 20 |
+
- **Multiple web workers = unpredictable scheduling.** If you scale to 2+ Uvicorn workers (or HF/Render restarts mid-request), a job started on worker A is invisible to worker B; nothing coordinates who's doing what.
|
| 21 |
+
- **No retry, no backoff, no dead-letter handling.** A transient failure (e.g. CACTUS timeout) just fails the job permanently.
|
| 22 |
+
- **No concurrency control.** Nothing stops 50 simultaneous Vina jobs from OOMing the box (this already happened once for unrelated reasons β see the Render OOM incident).
|
| 23 |
+
- **No visibility** beyond polling the DB row's `status` column β no queue depth, no stuck-job detection, no metrics.
|
| 24 |
+
|
| 25 |
+
We need a model where **job submission and job execution are decoupled processes**, and execution survives the web process dying.
|
| 26 |
+
|
| 27 |
+
---
|
| 28 |
+
|
| 29 |
+
## 2. Options considered
|
| 30 |
+
|
| 31 |
+
### Option A β Supabase-polling worker (no new infra)
|
| 32 |
+
|
| 33 |
+
A separate long-running process (or the same container, second entrypoint) polls `docking_jobs` / `sequencing_jobs` / `jobs` tables for rows with `status = 'queued'`, claims one, runs it, updates status.
|
| 34 |
+
|
| 35 |
+
**How claiming works (avoid double-processing):**
|
| 36 |
+
```sql
|
| 37 |
+
UPDATE docking_jobs
|
| 38 |
+
SET status = 'running', claimed_at = now(), claimed_by = $worker_id
|
| 39 |
+
WHERE id = (
|
| 40 |
+
SELECT id FROM docking_jobs
|
| 41 |
+
WHERE status = 'queued'
|
| 42 |
+
ORDER BY created_at ASC
|
| 43 |
+
LIMIT 1
|
| 44 |
+
FOR UPDATE SKIP LOCKED
|
| 45 |
+
)
|
| 46 |
+
RETURNING *;
|
| 47 |
+
```
|
| 48 |
+
`FOR UPDATE SKIP LOCKED` is the key primitive β Postgres lets concurrent workers each grab a different row without blocking each other or double-claiming. This works through PostgREST only if exposed via an RPC function (`supabase.rpc(...)`), since PostgREST doesn't expose raw `FOR UPDATE SKIP LOCKED` through its normal REST interface.
|
| 49 |
+
|
| 50 |
+
**Pros:**
|
| 51 |
+
- Zero new infrastructure β no Redis, no broker, nothing new to pay for or operate.
|
| 52 |
+
- Uses the database you already have and already trust.
|
| 53 |
+
- Easy to reason about: one table = one queue, visible with a normal `SELECT`.
|
| 54 |
+
- Natural fit for HF Spaces / Render, which don't easily support a second always-on process type (no separate "worker dyno" concept) β the worker can run as a background asyncio task *within* the same container using `asyncio.create_task` at startup instead of `run_in_executor` per-request, OR as a fully separate container/Space if you want isolation.
|
| 55 |
+
|
| 56 |
+
**Cons:**
|
| 57 |
+
- Polling has inherent latency (job sits `queued` until next poll tick β mitigate with a short interval, e.g. 2β3s, acceptable for 1β5 min jobs).
|
| 58 |
+
- You're hand-rolling retry/backoff/dead-letter logic that a real queue gives you for free.
|
| 59 |
+
- Postgres becomes your queue *and* your system of record β fine at this scale, but it doesn't scale to high job throughput (not a concern for your usage right now).
|
| 60 |
+
- Requires a Postgres function (`SECURITY DEFINER` RPC) to do the atomic claim safely β a bit of extra Supabase-side setup, not just application code.
|
| 61 |
+
|
| 62 |
+
### Option B β Redis + RQ (or Celery)
|
| 63 |
+
|
| 64 |
+
Add Redis as a message broker; jobs get pushed onto a Redis list/stream; RQ (simpler) or Celery (heavier, more features) workers consume from it.
|
| 65 |
+
|
| 66 |
+
**Pros:**
|
| 67 |
+
- Purpose-built for this β retries, backoff, dead-letter queues, concurrency limits, scheduling, and monitoring (RQ Dashboard / Flower for Celery) come out of the box.
|
| 68 |
+
- No polling latency β push-based.
|
| 69 |
+
- Scales cleanly if job volume grows a lot later.
|
| 70 |
+
|
| 71 |
+
**Cons:**
|
| 72 |
+
- **New infra dependency.** You already tried Redis for caching and noted "Redis unavailable β caching disabled" in your logs β meaning you don't currently have a reliable Redis instance running. This option requires actually standing one up (Upstash Redis, which you already have config fields for in `settings`, is the natural choice) and keeping it available, or job submission breaks.
|
| 73 |
+
- More moving parts to deploy/monitor/pay for, on top of an already multi-platform deployment (Vercel + HF/Render + Supabase).
|
| 74 |
+
- Given the recent debugging saga (Render OOM, HF migration, proxy misconfig), adding a 4th platform dependency raises operational risk right when things are stabilizing.
|
| 75 |
+
|
| 76 |
+
### Option C β External queue service (e.g. Supabase Edge Functions + pg_cron, or a managed queue like AWS SQS)
|
| 77 |
+
|
| 78 |
+
Considered and set aside: adds either vendor lock-in (SQS) or relies on Supabase-specific cron/edge tooling that's a bigger conceptual leap than A or B for the amount of job volume you actually have (single-digit concurrent jobs, not thousands/day).
|
| 79 |
+
|
| 80 |
+
---
|
| 81 |
+
|
| 82 |
+
## 3. Recommendation: **Option A β Supabase-polling worker**
|
| 83 |
+
|
| 84 |
+
Given:
|
| 85 |
+
- Current job volume is low (solo-developer / early-stage usage, not production traffic yet)
|
| 86 |
+
- You don't have a currently-working Redis instance (Upstash config exists but isn't active β "Redis unavailable" in every log)
|
| 87 |
+
- You're already juggling 3 deployment platforms (Vercel, HF Spaces, Supabase) and just spent a full day untangling a 4th (Render) β adding Redis as a 5th moving part right now works against stabilizing the stack
|
| 88 |
+
- `FOR UPDATE SKIP LOCKED` gives real correctness guarantees (no double-processing) without new infra
|
| 89 |
+
|
| 90 |
+
**Start with Option A.** If job volume grows enough that polling latency or Postgres-as-queue becomes a real bottleneck, migrating to Option B later is a contained, well-understood change (the job execution logic itself doesn't need to change β only how jobs get dispatched to the worker).
|
| 91 |
+
|
| 92 |
+
---
|
| 93 |
+
|
| 94 |
+
## 4. Proposed architecture (Option A)
|
| 95 |
+
|
| 96 |
+
### 4.1 Components
|
| 97 |
+
|
| 98 |
+
```
|
| 99 |
+
βββββββββββββββ ββββββββββββββββββββ ββββββββββββββββββ
|
| 100 |
+
β FastAPI β insert β docking_jobs / β poll β Worker process β
|
| 101 |
+
β web routes βββββββββΆβ sequencing_jobs βββββββββΆβ (same containerβ
|
| 102 |
+
β (unchanged β β tables β claim β or separate) β
|
| 103 |
+
β API surface)β β (Postgres queue) β β β
|
| 104 |
+
βββββββββββββββ ββββββββββββββββββββ ββββββββββββββββββ
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
- **Web routes** (`create_docking_job`, etc.) only ever `INSERT ... status='queued'` and return immediately. They never call `_run_docking_sync` directly anymore.
|
| 108 |
+
- **Worker** runs independently: a loop that claims one job at a time (or up to `N` concurrent, bounded), executes it, writes results.
|
| 109 |
+
- **Job tables** are the single source of truth and the queue itself β no separate broker.
|
| 110 |
+
|
| 111 |
+
### 4.2 Where the worker runs
|
| 112 |
+
|
| 113 |
+
Two sub-options, both viable:
|
| 114 |
+
|
| 115 |
+
**4.2a β In-process background task (simplest to ship first)**
|
| 116 |
+
At FastAPI startup (`@app.on_event("startup")` or lifespan), launch `asyncio.create_task(worker_loop())`. The worker runs inside the same Uvicorn process as the API.
|
| 117 |
+
|
| 118 |
+
- Pro: zero deployment changes, ships today.
|
| 119 |
+
- Con: still coupled to the web process's lifetime β a redeploy still interrupts in-flight jobs. **This does not fully solve the durability problem**, only the "multiple workers double-claim" problem. Acceptable as a *first step* since `FOR UPDATE SKIP LOCKED` at least makes retries safe (an interrupted job can be picked back up by the next process on restart, see Β§4.5).
|
| 120 |
+
|
| 121 |
+
**4.2b β Separate worker process/container (real fix)**
|
| 122 |
+
A second entrypoint (`python -m app.worker`) deployed as its own HF Space / Render service / process, running only the poll loop, no HTTP server. The web API and the worker scale and restart independently.
|
| 123 |
+
|
| 124 |
+
- Pro: actually decouples job execution from web process restarts β a Vercel/HF web redeploy no longer touches in-flight jobs.
|
| 125 |
+
- Con: a second thing to deploy and monitor (but same platform, no new vendor β just "one more Space" or "one more Render service").
|
| 126 |
+
|
| 127 |
+
**Recommendation:** ship 4.2a first (fast, and net positive vs. today even without full durability), then move to 4.2b once the claim/retry logic is proven correct. The application code is identical between the two β only how the process is launched differs.
|
| 128 |
+
|
| 129 |
+
### 4.3 Required Supabase-side changes
|
| 130 |
+
|
| 131 |
+
1. Add columns to `docking_jobs` / `sequencing_jobs` / `jobs`:
|
| 132 |
+
```sql
|
| 133 |
+
ALTER TABLE docking_jobs
|
| 134 |
+
ADD COLUMN IF NOT EXISTS claimed_at timestamptz,
|
| 135 |
+
ADD COLUMN IF NOT EXISTS claimed_by text,
|
| 136 |
+
ADD COLUMN IF NOT EXISTS attempts integer NOT NULL DEFAULT 0,
|
| 137 |
+
ADD COLUMN IF NOT EXISTS max_attempts integer NOT NULL DEFAULT 3;
|
| 138 |
+
```
|
| 139 |
+
|
| 140 |
+
2. Add a `SECURITY DEFINER` RPC function to do the atomic claim (PostgREST can't express `FOR UPDATE SKIP LOCKED` directly):
|
| 141 |
+
```sql
|
| 142 |
+
CREATE OR REPLACE FUNCTION claim_next_docking_job(worker_id text)
|
| 143 |
+
RETURNS docking_jobs
|
| 144 |
+
LANGUAGE plpgsql
|
| 145 |
+
SECURITY DEFINER
|
| 146 |
+
AS $$
|
| 147 |
+
DECLARE
|
| 148 |
+
job docking_jobs;
|
| 149 |
+
BEGIN
|
| 150 |
+
SELECT * INTO job
|
| 151 |
+
FROM docking_jobs
|
| 152 |
+
WHERE status = 'queued'
|
| 153 |
+
AND attempts < max_attempts
|
| 154 |
+
ORDER BY created_at ASC
|
| 155 |
+
LIMIT 1
|
| 156 |
+
FOR UPDATE SKIP LOCKED;
|
| 157 |
+
|
| 158 |
+
IF job.id IS NOT NULL THEN
|
| 159 |
+
UPDATE docking_jobs
|
| 160 |
+
SET status = 'running',
|
| 161 |
+
claimed_at = now(),
|
| 162 |
+
claimed_by = worker_id,
|
| 163 |
+
attempts = attempts + 1,
|
| 164 |
+
updated_at = now()
|
| 165 |
+
WHERE id = job.id
|
| 166 |
+
RETURNING * INTO job;
|
| 167 |
+
END IF;
|
| 168 |
+
|
| 169 |
+
RETURN job;
|
| 170 |
+
END;
|
| 171 |
+
$$;
|
| 172 |
+
```
|
| 173 |
+
Called from Python via `supabase.rpc("claim_next_docking_job", {"worker_id": WORKER_ID}).execute()`.
|
| 174 |
+
|
| 175 |
+
3. **Stuck-job recovery**: a job claimed but never completed (worker crashed mid-run) needs to be reclaimable. Add a periodic sweep (in the worker loop itself, every N iterations):
|
| 176 |
+
```sql
|
| 177 |
+
UPDATE docking_jobs
|
| 178 |
+
SET status = 'queued', claimed_at = NULL, claimed_by = NULL
|
| 179 |
+
WHERE status = 'running'
|
| 180 |
+
AND claimed_at < now() - interval '10 minutes';
|
| 181 |
+
```
|
| 182 |
+
(10 minutes should comfortably exceed the longest expected Vina run; tune as needed.)
|
| 183 |
+
|
| 184 |
+
### 4.4 Worker loop (pseudocode)
|
| 185 |
+
|
| 186 |
+
```python
|
| 187 |
+
WORKER_ID = f"{socket.gethostname()}-{os.getpid()}"
|
| 188 |
+
POLL_INTERVAL_SECONDS = 3
|
| 189 |
+
MAX_CONCURRENT_JOBS = 2 # bound memory/CPU use β was a real problem before (OOM)
|
| 190 |
+
|
| 191 |
+
async def worker_loop():
|
| 192 |
+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_JOBS)
|
| 193 |
+
sweep_counter = 0
|
| 194 |
+
|
| 195 |
+
while True:
|
| 196 |
+
sweep_counter += 1
|
| 197 |
+
if sweep_counter % 20 == 0: # every ~60s at 3s interval
|
| 198 |
+
await reclaim_stuck_jobs()
|
| 199 |
+
|
| 200 |
+
if semaphore.locked():
|
| 201 |
+
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
| 202 |
+
continue
|
| 203 |
+
|
| 204 |
+
job = await claim_next_job("docking_jobs")
|
| 205 |
+
if job is None:
|
| 206 |
+
await asyncio.sleep(POLL_INTERVAL_SECONDS)
|
| 207 |
+
continue
|
| 208 |
+
|
| 209 |
+
asyncio.create_task(run_claimed_job(job, semaphore))
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
async def run_claimed_job(job, semaphore):
|
| 213 |
+
async with semaphore:
|
| 214 |
+
try:
|
| 215 |
+
await asyncio.to_thread(_run_docking_sync, job["id"], job)
|
| 216 |
+
except Exception:
|
| 217 |
+
# _run_docking_sync already writes status='failed' internally;
|
| 218 |
+
# this catch is a last-resort safety net only.
|
| 219 |
+
logger.exception("Unhandled worker error for job %s", job["id"])
|
| 220 |
+
```
|
| 221 |
+
|
| 222 |
+
Note: `_run_docking_sync` stays synchronous internally (it shells out to Vina/Open Babel via `subprocess.run`, which is inherently blocking) β it's run via `asyncio.to_thread` from the worker loop rather than `loop.run_in_executor` from a request handler. Functionally similar today, but now driven by the worker's own loop instead of a web request's, and bounded by `MAX_CONCURRENT_JOBS`.
|
| 223 |
+
|
| 224 |
+
### 4.5 Retry semantics
|
| 225 |
+
|
| 226 |
+
- `attempts` increments on every claim (see RPC above).
|
| 227 |
+
- `_run_docking_sync`'s failure path checks `attempts >= max_attempts` before setting `status='failed'` permanently; otherwise sets back to `status='queued'` for another pickup.
|
| 228 |
+
- This requires a small change to the existing `except Exception as exc:` block in `_run_docking_sync` to branch on attempt count rather than unconditionally marking `failed`.
|
| 229 |
+
|
| 230 |
+
### 4.6 Traceback exposure (bundled fix, from the audit)
|
| 231 |
+
|
| 232 |
+
While touching this code path: stop writing `str(exc)` (which can include full tracebacks depending on exception type) into the `error` column that the frontend reads directly. Instead:
|
| 233 |
+
```python
|
| 234 |
+
except Exception as exc:
|
| 235 |
+
logger.exception("Docking job %s failed", job_id) # full traceback goes to server logs only
|
| 236 |
+
user_facing_error = "Docking failed during execution. Reference ID: " + job_id[:8]
|
| 237 |
+
...
|
| 238 |
+
```
|
| 239 |
+
|
| 240 |
+
---
|
| 241 |
+
|
| 242 |
+
## 5. Migration plan
|
| 243 |
+
|
| 244 |
+
1. Add new columns + RPC function to Supabase (docking_jobs first, then mirror to sequencing_jobs/jobs).
|
| 245 |
+
2. Implement `worker.py` with the claim/run/sweep loop described above.
|
| 246 |
+
3. Change `create_docking_job` to stop calling `run_in_executor` β just insert and return.
|
| 247 |
+
4. Launch the worker as an in-process background task (4.2a) behind a feature flag / env var (`ENABLE_INPROCESS_WORKER=true`) so it can be toggled off instantly if something's wrong.
|
| 248 |
+
5. Bake in `MAX_CONCURRENT_JOBS` from day one β directly addresses the earlier OOM incident.
|
| 249 |
+
6. Verify: submit several jobs concurrently, kill the process mid-job, confirm the stuck-job sweep reclaims it after the timeout.
|
| 250 |
+
7. Once stable, evaluate moving to a separate worker process (4.2b) if redeploy-safety becomes important (e.g. once this is used by others, not just you).
|
| 251 |
+
|
| 252 |
+
---
|
| 253 |
+
|
| 254 |
+
## 6. Explicitly out of scope for this doc
|
| 255 |
+
|
| 256 |
+
- Auth/ownership enforcement on job endpoints (separate P0 item, tracked separately)
|
| 257 |
+
- SSRF protection on `pdb_url` (separate P0 item)
|
| 258 |
+
- Chemically-accurate interaction detection (separate scientific-quality item)
|
| 259 |
+
- MD simulation, ADMET, DeepFRI β new feature modules, would each get queued through this same worker once it exists, but are not part of building the worker itself
|
| 260 |
+
|
| 261 |
+
---
|
| 262 |
+
|
| 263 |
+
## 7. Open questions for you
|
| 264 |
+
|
| 265 |
+
1. Are you okay running the worker in-process first (4.2a), accepting it's not *fully* durable yet, in exchange for shipping faster? Or do you want to go straight to a separate worker container (4.2b)?
|
| 266 |
+
2. `MAX_CONCURRENT_JOBS` β what's a safe number given HF's free-tier RAM (16GB) and the fact that each Vina job is a `subprocess`, not just memory in Python? I'd default to 2 and let you tune based on observed memory per job.
|
| 267 |
+
3. Do you want `sequencing_jobs` and the general `jobs` table migrated to the same worker pattern at the same time, or docking first as a proof of concept?
|