Spaces:
Running
Running
Upload folder using huggingface_hub
Browse files- app/ngs/contracts.py +7 -4
- app/ngs/orchestrator.py +39 -0
- app/routers/ngs_v2.py +5 -1
- clean/HUM0001_R1.fastq.clean.fastq.gz +0 -0
- clean/SAMPLE_001_R1.fastq.clean.fastq.gz +0 -0
- tests/test_ngs_v2_router.py +11 -1
app/ngs/contracts.py
CHANGED
|
@@ -15,8 +15,8 @@ A stage result looks like:
|
|
| 15 |
"step": "alignment",
|
| 16 |
"tool": "bwa-mem2",
|
| 17 |
"version": "2.2.1",
|
| 18 |
-
"
|
| 19 |
-
"
|
| 20 |
"qc": {
|
| 21 |
"mapping_rate": {"value": 97.8, "status": "PASS"},
|
| 22 |
"proper_pair": {"value": 96.2, "status": "PASS"}
|
|
@@ -233,8 +233,11 @@ class StageResult:
|
|
| 233 |
"step": self.step,
|
| 234 |
"tool": self.tool,
|
| 235 |
"version": self.version,
|
| 236 |
-
|
| 237 |
-
|
|
|
|
|
|
|
|
|
|
| 238 |
"qc": self.qc.to_dict() if self.qc else None,
|
| 239 |
"decision": self.decision.value,
|
| 240 |
"data": self.data,
|
|
|
|
| 15 |
"step": "alignment",
|
| 16 |
"tool": "bwa-mem2",
|
| 17 |
"version": "2.2.1",
|
| 18 |
+
"inputs": ["clean_R1", "clean_R2"],
|
| 19 |
+
"outputs": ["sample.bam"],
|
| 20 |
"qc": {
|
| 21 |
"mapping_rate": {"value": 97.8, "status": "PASS"},
|
| 22 |
"proper_pair": {"value": 96.2, "status": "PASS"}
|
|
|
|
| 233 |
"step": self.step,
|
| 234 |
"tool": self.tool,
|
| 235 |
"version": self.version,
|
| 236 |
+
# Keep the runtime report aligned with the public stage-contract endpoint and
|
| 237 |
+
# frontend API type. The old singular keys made otherwise successful runs crash
|
| 238 |
+
# when the evidence-chain UI tried to render ``stage.outputs.join(...)``.
|
| 239 |
+
"inputs": self.inputs,
|
| 240 |
+
"outputs": self.outputs,
|
| 241 |
"qc": self.qc.to_dict() if self.qc else None,
|
| 242 |
"decision": self.decision.value,
|
| 243 |
"data": self.data,
|
app/ngs/orchestrator.py
CHANGED
|
@@ -21,6 +21,7 @@ from __future__ import annotations
|
|
| 21 |
|
| 22 |
import json
|
| 23 |
import logging
|
|
|
|
| 24 |
from typing import Callable, Optional
|
| 25 |
|
| 26 |
from app.ngs.contracts import (
|
|
@@ -80,9 +81,47 @@ class Pipeline:
|
|
| 80 |
self.name, contract.step)
|
| 81 |
break
|
| 82 |
|
|
|
|
|
|
|
| 83 |
# Merge any structured payload from the last successful stage into state for consumers.
|
| 84 |
return self.report()
|
| 85 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
def report(self) -> dict:
|
| 87 |
return {
|
| 88 |
"pipeline": self.name,
|
|
|
|
| 21 |
|
| 22 |
import json
|
| 23 |
import logging
|
| 24 |
+
import os
|
| 25 |
from typing import Callable, Optional
|
| 26 |
|
| 27 |
from app.ngs.contracts import (
|
|
|
|
| 81 |
self.name, contract.step)
|
| 82 |
break
|
| 83 |
|
| 84 |
+
self.provenance = self._build_provenance(sample)
|
| 85 |
+
|
| 86 |
# Merge any structured payload from the last successful stage into state for consumers.
|
| 87 |
return self.report()
|
| 88 |
|
| 89 |
+
def _build_provenance(self, sample: dict) -> dict:
|
| 90 |
+
"""Build a compact, deterministic audit record from facts observed by this run."""
|
| 91 |
+
metadata = sample.get("metadata") or {}
|
| 92 |
+
input_stage = next((r for r in self.results if r.step == "input_validation"), None)
|
| 93 |
+
checksums = input_stage.data.get("checksums", {}) if input_stage else {}
|
| 94 |
+
files = []
|
| 95 |
+
for path in sample.get("files") or []:
|
| 96 |
+
item = {"name": os.path.basename(path)}
|
| 97 |
+
checksum = checksums.get(path) or checksums.get(os.path.basename(path))
|
| 98 |
+
if checksum:
|
| 99 |
+
item["checksum"] = {"algorithm": "md5", "value": checksum}
|
| 100 |
+
files.append(item)
|
| 101 |
+
|
| 102 |
+
reference = self.state.get("reference", {}).get("declared") or {
|
| 103 |
+
"id": sample.get("reference")
|
| 104 |
+
}
|
| 105 |
+
return {
|
| 106 |
+
"schema_version": "1.0",
|
| 107 |
+
"pipeline": {"name": self.name, "version": self.version},
|
| 108 |
+
"analysis": {
|
| 109 |
+
"assay": sample.get("assay"),
|
| 110 |
+
"sample_type": sample.get("sample_type"),
|
| 111 |
+
"platform": metadata.get("platform"),
|
| 112 |
+
"demonstration_data": bool(
|
| 113 |
+
sample.get("demonstration_data") or metadata.get("demonstration_data")
|
| 114 |
+
),
|
| 115 |
+
"synthetic_reference": bool(sample.get("synthetic_reference")),
|
| 116 |
+
},
|
| 117 |
+
"inputs": files,
|
| 118 |
+
"reference": reference,
|
| 119 |
+
"tools": [
|
| 120 |
+
{"stage": r.step, "name": r.tool, "version": r.version}
|
| 121 |
+
for r in self.results
|
| 122 |
+
],
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
def report(self) -> dict:
|
| 126 |
return {
|
| 127 |
"pipeline": self.name,
|
app/routers/ngs_v2.py
CHANGED
|
@@ -258,9 +258,13 @@ def analyze(payload: AnalyzeRequest):
|
|
| 258 |
"assay": assay,
|
| 259 |
"sample_type": payload.sample_type or detection["sample_type"],
|
| 260 |
"metadata": {**(payload.metadata or {}), **({"demo_profile": payload.demo_profile} if payload.demo_profile else {})},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
"reads": list(reads_all.values())[0] if reads_all else [],
|
| 262 |
}
|
| 263 |
-
use_synthetic_reference =
|
| 264 |
if use_synthetic_reference and sample["reads"]:
|
| 265 |
sample["reference_seq"] = _synthetic_reference(sample["reads"])
|
| 266 |
sample["contig"] = "chr1"
|
|
|
|
| 258 |
"assay": assay,
|
| 259 |
"sample_type": payload.sample_type or detection["sample_type"],
|
| 260 |
"metadata": {**(payload.metadata or {}), **({"demo_profile": payload.demo_profile} if payload.demo_profile else {})},
|
| 261 |
+
"demonstration_data": bool(
|
| 262 |
+
payload.demo_profile or (payload.metadata or {}).get("demonstration_data")
|
| 263 |
+
),
|
| 264 |
+
"synthetic_reference": bool(payload.synthetic_reference or payload.demo_profile),
|
| 265 |
"reads": list(reads_all.values())[0] if reads_all else [],
|
| 266 |
}
|
| 267 |
+
use_synthetic_reference = sample["synthetic_reference"]
|
| 268 |
if use_synthetic_reference and sample["reads"]:
|
| 269 |
sample["reference_seq"] = _synthetic_reference(sample["reads"])
|
| 270 |
sample["contig"] = "chr1"
|
clean/HUM0001_R1.fastq.clean.fastq.gz
ADDED
|
Binary file (3.08 kB). View file
|
|
|
clean/SAMPLE_001_R1.fastq.clean.fastq.gz
ADDED
|
Binary file (3.74 kB). View file
|
|
|
tests/test_ngs_v2_router.py
CHANGED
|
@@ -72,6 +72,9 @@ def test_analyze_runs_full_dag_through_final_gate(client, tmp_path):
|
|
| 72 |
stages = body["pipeline"]["stages"]
|
| 73 |
steps = [s["step"] for s in stages]
|
| 74 |
assert len(steps) == 21
|
|
|
|
|
|
|
|
|
|
| 75 |
# The pipeline should NOT have hard-stopped on a blocking gate for a genuine read set.
|
| 76 |
assert body["pipeline"]["pipeline_status"] in ("PASS", "WARN")
|
| 77 |
|
|
@@ -80,6 +83,14 @@ def test_analyze_runs_full_dag_through_final_gate(client, tmp_path):
|
|
| 80 |
assert gate["step"] == "final_gate"
|
| 81 |
assert gate["decision"] in ("CONTINUE", "CONTINUE_WITH_WARNING")
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
def test_analyze_emits_igv_tracks(client, tmp_path):
|
| 85 |
r1, _ = _write_fastq(tmp_path, "HUM0001_R1.fastq.gz", 200, seed=7)
|
|
@@ -109,4 +120,3 @@ def test_analyze_emits_igv_tracks(client, tmp_path):
|
|
| 109 |
assert len(cols) == 8
|
| 110 |
assert viz["n_variants"] == len(var_lines)
|
| 111 |
assert viz["locus"]
|
| 112 |
-
|
|
|
|
| 72 |
stages = body["pipeline"]["stages"]
|
| 73 |
steps = [s["step"] for s in stages]
|
| 74 |
assert len(steps) == 21
|
| 75 |
+
assert all(isinstance(s["inputs"], list) for s in stages)
|
| 76 |
+
assert all(isinstance(s["outputs"], list) for s in stages)
|
| 77 |
+
assert all("input" not in s and "output" not in s for s in stages)
|
| 78 |
# The pipeline should NOT have hard-stopped on a blocking gate for a genuine read set.
|
| 79 |
assert body["pipeline"]["pipeline_status"] in ("PASS", "WARN")
|
| 80 |
|
|
|
|
| 83 |
assert gate["step"] == "final_gate"
|
| 84 |
assert gate["decision"] in ("CONTINUE", "CONTINUE_WITH_WARNING")
|
| 85 |
|
| 86 |
+
provenance = body["pipeline"]["provenance"]
|
| 87 |
+
assert provenance["schema_version"] == "1.0"
|
| 88 |
+
assert provenance["pipeline"] == {"name": "WGS-germline", "version": "0.1.0"}
|
| 89 |
+
assert provenance["analysis"]["synthetic_reference"] is True
|
| 90 |
+
assert len(provenance["inputs"]) == 2
|
| 91 |
+
assert all(item["checksum"]["algorithm"] == "md5" for item in provenance["inputs"])
|
| 92 |
+
assert len(provenance["tools"]) == 21
|
| 93 |
+
|
| 94 |
|
| 95 |
def test_analyze_emits_igv_tracks(client, tmp_path):
|
| 96 |
r1, _ = _write_fastq(tmp_path, "HUM0001_R1.fastq.gz", 200, seed=7)
|
|
|
|
| 120 |
assert len(cols) == 8
|
| 121 |
assert viz["n_variants"] == len(var_lines)
|
| 122 |
assert viz["locus"]
|
|
|