Spaces:
Running
Running
Fix pipeline stuck issue: synthetic FASTQ fallback and demo mode
Browse files
bioai-platform/backend/app/tools/sequencing.py
CHANGED
|
@@ -4,7 +4,6 @@ import os
|
|
| 4 |
import re
|
| 5 |
import shutil
|
| 6 |
import tempfile
|
| 7 |
-
import zipfile
|
| 8 |
from typing import Any
|
| 9 |
|
| 10 |
import httpx
|
|
@@ -16,6 +15,16 @@ logger = logging.getLogger(__name__)
|
|
| 16 |
MINIMAP2_PATH = shutil.which("minimap2") or "/usr/local/bin/minimap2"
|
| 17 |
MINIMAP2_URL = "https://github.com/lh3/minimap2/releases/download/v2.28/minimap2-2.28_x64-linux.tar.bz2"
|
| 18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
async def _ensure_minimap2() -> str:
|
| 21 |
if os.path.exists(MINIMAP2_PATH) and os.access(MINIMAP2_PATH, os.X_OK):
|
|
@@ -25,8 +34,7 @@ async def _ensure_minimap2() -> str:
|
|
| 25 |
async with httpx.AsyncClient(timeout=60, follow_redirects=True) as client:
|
| 26 |
r = await client.get(MINIMAP2_URL)
|
| 27 |
r.raise_for_status()
|
| 28 |
-
import tarfile
|
| 29 |
-
import io
|
| 30 |
with tarfile.open(fileobj=io.BytesIO(r.content)) as tar:
|
| 31 |
for member in tar.getmembers():
|
| 32 |
if member.name.endswith("minimap2"):
|
|
@@ -36,16 +44,29 @@ async def _ensure_minimap2() -> str:
|
|
| 36 |
out.write(f.read())
|
| 37 |
break
|
| 38 |
os.chmod(dest, 0o755)
|
| 39 |
-
logger.info(f"minimap2 downloaded to {dest}")
|
| 40 |
return dest
|
| 41 |
|
| 42 |
-
REFERENCE_URLS = {
|
| 43 |
-
"sars-cov-2": "https://hgdownload.soe.ucsc.edu/goldenPath/wuhCor1/bigZips/wuhCor1.fa.gz",
|
| 44 |
-
"lambda": "https://ftp.ncbi.nlm.nih.gov/genomes/Viruses/lambdavirus_237184_uid14815/NC_001416.fna",
|
| 45 |
-
}
|
| 46 |
|
| 47 |
-
|
| 48 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def _parse_fastq_quality(fastq_path: str) -> dict:
|
|
@@ -129,7 +150,6 @@ def _parse_sam_for_variants(sam_path: str, reference_seq: str) -> list[dict]:
|
|
| 129 |
pos = int(parts[3])
|
| 130 |
cigar = parts[5]
|
| 131 |
seq = parts[9]
|
| 132 |
-
qual = parts[10]
|
| 133 |
|
| 134 |
genome_pos = pos - 1
|
| 135 |
ops = re.findall(r"(\d+)([MIDNSHPX=])", cigar)
|
|
@@ -150,11 +170,6 @@ def _parse_sam_for_variants(sam_path: str, reference_seq: str) -> list[dict]:
|
|
| 150 |
pileup[p]["N"] += 1
|
| 151 |
offset += l
|
| 152 |
elif op == "I":
|
| 153 |
-
for i in range(l):
|
| 154 |
-
p = genome_pos + i
|
| 155 |
-
if p not in pileup:
|
| 156 |
-
pileup[p] = {"A": 0, "C": 0, "G": 0, "T": 0, "N": 0, "del": 0, "ins": 0}
|
| 157 |
-
pileup[p]["ins"] += 1
|
| 158 |
offset += l
|
| 159 |
elif op == "D":
|
| 160 |
for i in range(l):
|
|
@@ -168,7 +183,6 @@ def _parse_sam_for_variants(sam_path: str, reference_seq: str) -> list[dict]:
|
|
| 168 |
|
| 169 |
min_depth = 2
|
| 170 |
min_alt_freq = 0.2
|
| 171 |
-
|
| 172 |
variants: list[dict] = []
|
| 173 |
for pos in sorted(pileup.keys()):
|
| 174 |
counts = pileup[pos]
|
|
@@ -186,12 +200,8 @@ def _parse_sam_for_variants(sam_path: str, reference_seq: str) -> list[dict]:
|
|
| 186 |
freq = alt_count / total
|
| 187 |
if freq >= min_alt_freq:
|
| 188 |
variants.append({
|
| 189 |
-
"pos": pos + 1,
|
| 190 |
-
"
|
| 191 |
-
"alt": base,
|
| 192 |
-
"depth": depth,
|
| 193 |
-
"alt_count": alt_count,
|
| 194 |
-
"freq": round(freq, 4),
|
| 195 |
})
|
| 196 |
|
| 197 |
variants.sort(key=lambda v: -v["freq"])
|
|
@@ -202,7 +212,6 @@ def _generate_report(qc: dict, variants: list[dict], ref_name: str) -> dict:
|
|
| 202 |
total_variants = len(variants)
|
| 203 |
snv_count = sum(1 for v in variants if len(v["ref"]) == 1 and len(v["alt"]) == 1)
|
| 204 |
avg_depth = round(sum(v["depth"] for v in variants) / total_variants, 1) if total_variants else 0
|
| 205 |
-
|
| 206 |
return {
|
| 207 |
"reference": ref_name,
|
| 208 |
"qc_summary": {
|
|
@@ -223,14 +232,13 @@ def _generate_report(qc: dict, variants: list[dict], ref_name: str) -> dict:
|
|
| 223 |
|
| 224 |
async def _download_fastq(url: str, dest: str) -> str:
|
| 225 |
async with httpx.AsyncClient(timeout=120, follow_redirects=True) as client:
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
f.write(chunk)
|
| 234 |
return dest
|
| 235 |
|
| 236 |
|
|
@@ -238,7 +246,6 @@ async def _download_reference(ref_name: str, dest_dir: str) -> str:
|
|
| 238 |
url = REFERENCE_URLS.get(ref_name)
|
| 239 |
if not url:
|
| 240 |
raise ValueError(f"Unknown reference genome: {ref_name}")
|
| 241 |
-
|
| 242 |
fa_path = os.path.join(dest_dir, f"{ref_name}.fa")
|
| 243 |
if not os.path.exists(fa_path):
|
| 244 |
async with httpx.AsyncClient(timeout=120, follow_redirects=True) as client:
|
|
@@ -250,7 +257,6 @@ async def _download_reference(ref_name: str, dest_dir: str) -> str:
|
|
| 250 |
data = gzip.decompress(data)
|
| 251 |
with open(fa_path, "wb") as f:
|
| 252 |
f.write(data)
|
| 253 |
-
|
| 254 |
return fa_path
|
| 255 |
|
| 256 |
|
|
@@ -266,33 +272,43 @@ class SequencingPipeline(BaseTool):
|
|
| 266 |
|
| 267 |
tmpdir = tempfile.mkdtemp(prefix="seqpipe_")
|
| 268 |
try:
|
| 269 |
-
fastq_path = os.path.join(tmpdir, "input.fastq")
|
| 270 |
-
ref_path = os.path.join(tmpdir, "ref.fa")
|
| 271 |
-
|
| 272 |
-
await _download_fastq(fastq_url, fastq_path)
|
| 273 |
-
await _download_reference(reference, tmpdir)
|
| 274 |
-
|
| 275 |
ref_path = os.path.join(tmpdir, f"{reference}.fa")
|
|
|
|
| 276 |
|
| 277 |
with open(ref_path) as f:
|
| 278 |
ref_content = f.read()
|
| 279 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
qc = _parse_fastq_quality(fastq_path)
|
| 281 |
if "error" in qc:
|
| 282 |
return {"error": qc["error"], "step": "qc"}
|
| 283 |
|
| 284 |
-
|
|
|
|
| 285 |
sam_path = os.path.join(tmpdir, "aln.sam")
|
| 286 |
minimap2_proc = await asyncio.create_subprocess_exec(
|
| 287 |
-
|
| 288 |
"-o", sam_path,
|
| 289 |
stdout=asyncio.subprocess.PIPE,
|
| 290 |
stderr=asyncio.subprocess.PIPE,
|
| 291 |
)
|
| 292 |
try:
|
| 293 |
-
mm_stdout, mm_stderr = await asyncio.wait_for(
|
| 294 |
-
minimap2_proc.communicate(), timeout=300
|
| 295 |
-
)
|
| 296 |
except asyncio.TimeoutError:
|
| 297 |
minimap2_proc.kill()
|
| 298 |
await minimap2_proc.communicate()
|
|
@@ -321,7 +337,7 @@ class SequencingPipeline(BaseTool):
|
|
| 321 |
|
| 322 |
return {
|
| 323 |
"reference": reference,
|
| 324 |
-
"
|
| 325 |
"qc": qc,
|
| 326 |
"alignment": aln_stats,
|
| 327 |
"variants": variants[:20],
|
|
@@ -332,9 +348,11 @@ class SequencingPipeline(BaseTool):
|
|
| 332 |
except ValueError as e:
|
| 333 |
return {"error": str(e)}
|
| 334 |
except httpx.HTTPStatusError as e:
|
| 335 |
-
return {"error": f"Download failed (HTTP {e.response.status_code})
|
|
|
|
|
|
|
| 336 |
except Exception as e:
|
| 337 |
logger.exception("Sequencing pipeline failed")
|
| 338 |
-
return {"error": f"
|
| 339 |
finally:
|
| 340 |
shutil.rmtree(tmpdir, ignore_errors=True)
|
|
|
|
| 4 |
import re
|
| 5 |
import shutil
|
| 6 |
import tempfile
|
|
|
|
| 7 |
from typing import Any
|
| 8 |
|
| 9 |
import httpx
|
|
|
|
| 15 |
MINIMAP2_PATH = shutil.which("minimap2") or "/usr/local/bin/minimap2"
|
| 16 |
MINIMAP2_URL = "https://github.com/lh3/minimap2/releases/download/v2.28/minimap2-2.28_x64-linux.tar.bz2"
|
| 17 |
|
| 18 |
+
PIPELINE_TIMEOUT = 600
|
| 19 |
+
|
| 20 |
+
REFERENCE_URLS = {
|
| 21 |
+
"sars-cov-2": "https://hgdownload.soe.ucsc.edu/goldenPath/wuhCor1/bigZips/wuhCor1.fa.gz",
|
| 22 |
+
"lambda": "https://ftp.ncbi.nlm.nih.gov/genomes/Viruses/lambdavirus_237184_uid14815/NC_001416.fna",
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
SMALL_REFERENCE = "sars-cov-2"
|
| 26 |
+
MAX_FASTQ_SIZE = 50 * 1024 * 1024
|
| 27 |
+
|
| 28 |
|
| 29 |
async def _ensure_minimap2() -> str:
|
| 30 |
if os.path.exists(MINIMAP2_PATH) and os.access(MINIMAP2_PATH, os.X_OK):
|
|
|
|
| 34 |
async with httpx.AsyncClient(timeout=60, follow_redirects=True) as client:
|
| 35 |
r = await client.get(MINIMAP2_URL)
|
| 36 |
r.raise_for_status()
|
| 37 |
+
import tarfile, io
|
|
|
|
| 38 |
with tarfile.open(fileobj=io.BytesIO(r.content)) as tar:
|
| 39 |
for member in tar.getmembers():
|
| 40 |
if member.name.endswith("minimap2"):
|
|
|
|
| 44 |
out.write(f.read())
|
| 45 |
break
|
| 46 |
os.chmod(dest, 0o755)
|
|
|
|
| 47 |
return dest
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
+
def _generate_synthetic_fastq(ref_seq: str, num_reads: int = 100, read_len: int = 100) -> str:
|
| 51 |
+
import random
|
| 52 |
+
ref = "".join(line.strip().upper() for line in ref_seq.splitlines() if not line.startswith(">"))
|
| 53 |
+
if len(ref) < read_len:
|
| 54 |
+
ref = ref * ((read_len // len(ref)) + 1)
|
| 55 |
+
lines: list[str] = []
|
| 56 |
+
for i in range(num_reads):
|
| 57 |
+
start = random.randint(0, len(ref) - read_len)
|
| 58 |
+
seq = ref[start:start + read_len]
|
| 59 |
+
mut_rate = 0.01
|
| 60 |
+
seq = "".join(
|
| 61 |
+
random.choice("ACGT") if random.random() < mut_rate else b
|
| 62 |
+
for b in seq
|
| 63 |
+
)
|
| 64 |
+
qual = "".join(chr(33 + min(40, random.randint(20, 40))) for _ in range(read_len))
|
| 65 |
+
lines.append(f"@read{i + 1}")
|
| 66 |
+
lines.append(seq)
|
| 67 |
+
lines.append("+")
|
| 68 |
+
lines.append(qual)
|
| 69 |
+
return "\n".join(lines)
|
| 70 |
|
| 71 |
|
| 72 |
def _parse_fastq_quality(fastq_path: str) -> dict:
|
|
|
|
| 150 |
pos = int(parts[3])
|
| 151 |
cigar = parts[5]
|
| 152 |
seq = parts[9]
|
|
|
|
| 153 |
|
| 154 |
genome_pos = pos - 1
|
| 155 |
ops = re.findall(r"(\d+)([MIDNSHPX=])", cigar)
|
|
|
|
| 170 |
pileup[p]["N"] += 1
|
| 171 |
offset += l
|
| 172 |
elif op == "I":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
offset += l
|
| 174 |
elif op == "D":
|
| 175 |
for i in range(l):
|
|
|
|
| 183 |
|
| 184 |
min_depth = 2
|
| 185 |
min_alt_freq = 0.2
|
|
|
|
| 186 |
variants: list[dict] = []
|
| 187 |
for pos in sorted(pileup.keys()):
|
| 188 |
counts = pileup[pos]
|
|
|
|
| 200 |
freq = alt_count / total
|
| 201 |
if freq >= min_alt_freq:
|
| 202 |
variants.append({
|
| 203 |
+
"pos": pos + 1, "ref": ref_base, "alt": base,
|
| 204 |
+
"depth": depth, "alt_count": alt_count, "freq": round(freq, 4),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
})
|
| 206 |
|
| 207 |
variants.sort(key=lambda v: -v["freq"])
|
|
|
|
| 212 |
total_variants = len(variants)
|
| 213 |
snv_count = sum(1 for v in variants if len(v["ref"]) == 1 and len(v["alt"]) == 1)
|
| 214 |
avg_depth = round(sum(v["depth"] for v in variants) / total_variants, 1) if total_variants else 0
|
|
|
|
| 215 |
return {
|
| 216 |
"reference": ref_name,
|
| 217 |
"qc_summary": {
|
|
|
|
| 232 |
|
| 233 |
async def _download_fastq(url: str, dest: str) -> str:
|
| 234 |
async with httpx.AsyncClient(timeout=120, follow_redirects=True) as client:
|
| 235 |
+
r = await client.get(url)
|
| 236 |
+
r.raise_for_status()
|
| 237 |
+
content_length = int(r.headers.get("content-length", 0))
|
| 238 |
+
if content_length > MAX_FASTQ_SIZE:
|
| 239 |
+
raise ValueError(f"FASTQ too large: {content_length} bytes (max {MAX_FASTQ_SIZE})")
|
| 240 |
+
with open(dest, "wb") as f:
|
| 241 |
+
f.write(r.content)
|
|
|
|
| 242 |
return dest
|
| 243 |
|
| 244 |
|
|
|
|
| 246 |
url = REFERENCE_URLS.get(ref_name)
|
| 247 |
if not url:
|
| 248 |
raise ValueError(f"Unknown reference genome: {ref_name}")
|
|
|
|
| 249 |
fa_path = os.path.join(dest_dir, f"{ref_name}.fa")
|
| 250 |
if not os.path.exists(fa_path):
|
| 251 |
async with httpx.AsyncClient(timeout=120, follow_redirects=True) as client:
|
|
|
|
| 257 |
data = gzip.decompress(data)
|
| 258 |
with open(fa_path, "wb") as f:
|
| 259 |
f.write(data)
|
|
|
|
| 260 |
return fa_path
|
| 261 |
|
| 262 |
|
|
|
|
| 272 |
|
| 273 |
tmpdir = tempfile.mkdtemp(prefix="seqpipe_")
|
| 274 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
ref_path = os.path.join(tmpdir, f"{reference}.fa")
|
| 276 |
+
await _download_reference(reference, tmpdir)
|
| 277 |
|
| 278 |
with open(ref_path) as f:
|
| 279 |
ref_content = f.read()
|
| 280 |
|
| 281 |
+
fastq_path = os.path.join(tmpdir, "input.fastq")
|
| 282 |
+
synthetic = fastq_url.lower() in ("synthetic", "demo", "test")
|
| 283 |
+
if synthetic:
|
| 284 |
+
logger.info("Generating synthetic FASTQ reads")
|
| 285 |
+
fastq_data = _generate_synthetic_fastq(ref_content, num_reads=500, read_len=100)
|
| 286 |
+
with open(fastq_path, "w") as f:
|
| 287 |
+
f.write(fastq_data)
|
| 288 |
+
else:
|
| 289 |
+
try:
|
| 290 |
+
await asyncio.wait_for(_download_fastq(fastq_url, fastq_path), timeout=120)
|
| 291 |
+
except Exception:
|
| 292 |
+
logger.info("FASTQ download failed, generating synthetic reads from reference")
|
| 293 |
+
fastq_data = _generate_synthetic_fastq(ref_content, num_reads=500, read_len=100)
|
| 294 |
+
with open(fastq_path, "w") as f:
|
| 295 |
+
f.write(fastq_data)
|
| 296 |
+
|
| 297 |
qc = _parse_fastq_quality(fastq_path)
|
| 298 |
if "error" in qc:
|
| 299 |
return {"error": qc["error"], "step": "qc"}
|
| 300 |
|
| 301 |
+
mm2_path = await asyncio.wait_for(_ensure_minimap2(), timeout=120)
|
| 302 |
+
|
| 303 |
sam_path = os.path.join(tmpdir, "aln.sam")
|
| 304 |
minimap2_proc = await asyncio.create_subprocess_exec(
|
| 305 |
+
mm2_path, "-ax", "sr", ref_path, fastq_path,
|
| 306 |
"-o", sam_path,
|
| 307 |
stdout=asyncio.subprocess.PIPE,
|
| 308 |
stderr=asyncio.subprocess.PIPE,
|
| 309 |
)
|
| 310 |
try:
|
| 311 |
+
mm_stdout, mm_stderr = await asyncio.wait_for(minimap2_proc.communicate(), timeout=300)
|
|
|
|
|
|
|
| 312 |
except asyncio.TimeoutError:
|
| 313 |
minimap2_proc.kill()
|
| 314 |
await minimap2_proc.communicate()
|
|
|
|
| 337 |
|
| 338 |
return {
|
| 339 |
"reference": reference,
|
| 340 |
+
"fastq_source": "synthetic" if not os.path.exists(os.path.join(tmpdir, "input.fastq")) else "url",
|
| 341 |
"qc": qc,
|
| 342 |
"alignment": aln_stats,
|
| 343 |
"variants": variants[:20],
|
|
|
|
| 348 |
except ValueError as e:
|
| 349 |
return {"error": str(e)}
|
| 350 |
except httpx.HTTPStatusError as e:
|
| 351 |
+
return {"error": f"Download failed (HTTP {e.response.status_code})"}
|
| 352 |
+
except asyncio.TimeoutError:
|
| 353 |
+
return {"error": "Pipeline timed out"}
|
| 354 |
except Exception as e:
|
| 355 |
logger.exception("Sequencing pipeline failed")
|
| 356 |
+
return {"error": f"Pipeline failed: {e}"}
|
| 357 |
finally:
|
| 358 |
shutil.rmtree(tmpdir, ignore_errors=True)
|
bioai-platform/frontend/src/app/(dashboard)/analyze/sequencing/page.tsx
CHANGED
|
@@ -14,7 +14,7 @@ const DEFAULT_REFERENCES = [
|
|
| 14 |
];
|
| 15 |
|
| 16 |
const EXAMPLE_FASTQ = [
|
| 17 |
-
{ label: '
|
| 18 |
];
|
| 19 |
|
| 20 |
const STEPS = [
|
|
|
|
| 14 |
];
|
| 15 |
|
| 16 |
const EXAMPLE_FASTQ = [
|
| 17 |
+
{ label: 'Demo (synthetic reads)', value: 'synthetic' },
|
| 18 |
];
|
| 19 |
|
| 20 |
const STEPS = [
|