Upload export_routes.py
Browse files- backend/app/export_routes.py +152 -0
backend/app/export_routes.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CSV downloads for the screening and extraction phase."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import csv
|
| 5 |
+
import io
|
| 6 |
+
import re
|
| 7 |
+
from typing import Any, Iterable
|
| 8 |
+
|
| 9 |
+
from fastapi import APIRouter, HTTPException
|
| 10 |
+
from fastapi.responses import Response
|
| 11 |
+
|
| 12 |
+
from app import storage
|
| 13 |
+
from app.schemas import ScreeningRole, ScreeningStage
|
| 14 |
+
|
| 15 |
+
router = APIRouter(prefix="/api/v1/protocols/{protocol_id}/export", tags=["export"])
|
| 16 |
+
|
| 17 |
+
# Spreadsheet programs run cells starting with these characters as formulas.
|
| 18 |
+
_FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _cell(value: Any) -> str:
|
| 22 |
+
if value is None:
|
| 23 |
+
return ""
|
| 24 |
+
if isinstance(value, (list, tuple)):
|
| 25 |
+
text = "; ".join(_cell(v) for v in value)
|
| 26 |
+
elif hasattr(value, "value"): # enums
|
| 27 |
+
text = str(value.value)
|
| 28 |
+
elif hasattr(value, "isoformat"): # datetimes
|
| 29 |
+
text = value.isoformat()
|
| 30 |
+
else:
|
| 31 |
+
text = str(value)
|
| 32 |
+
return "'" + text if text.startswith(_FORMULA_PREFIXES) else text
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _csv(filename: str, header: list[str], rows: Iterable[list[Any]]) -> Response:
|
| 36 |
+
buffer = io.StringIO()
|
| 37 |
+
writer = csv.writer(buffer)
|
| 38 |
+
writer.writerow(header)
|
| 39 |
+
for row in rows:
|
| 40 |
+
writer.writerow([_cell(c) for c in row])
|
| 41 |
+
return Response(
|
| 42 |
+
# The BOM makes Excel read accented characters correctly.
|
| 43 |
+
content="\ufeff" + buffer.getvalue(),
|
| 44 |
+
media_type="text/csv; charset=utf-8",
|
| 45 |
+
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _slug(title: str) -> str:
|
| 50 |
+
return re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-")[:50] or "review"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
async def _protocol_or_404(protocol_id: str):
|
| 54 |
+
protocol = await storage.get_protocol(protocol_id)
|
| 55 |
+
if protocol is None:
|
| 56 |
+
raise HTTPException(status_code=404, detail="Protocol not found")
|
| 57 |
+
return protocol
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _latest_per_reviewer(entries: list) -> dict:
|
| 61 |
+
latest: dict = {}
|
| 62 |
+
for e in entries:
|
| 63 |
+
current = latest.get(e.reviewer)
|
| 64 |
+
if current is None or e.decided_at >= current.decided_at:
|
| 65 |
+
latest[e.reviewer] = e
|
| 66 |
+
return latest
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _stage_summary(entries: list, reviewers_required: int) -> tuple[str, str, str]:
|
| 70 |
+
"""(each reviewer's decision, final decision, exclusion reasons)."""
|
| 71 |
+
reviews = _latest_per_reviewer([e for e in entries if e.role == ScreeningRole.REVIEW])
|
| 72 |
+
adjudications = [e for e in entries if e.role == ScreeningRole.ADJUDICATION]
|
| 73 |
+
per_reviewer = "; ".join(f"{r}: {e.decision.value}" for r, e in reviews.items())
|
| 74 |
+
reasons = "; ".join(
|
| 75 |
+
dict.fromkeys(e.reason for e in reviews.values() if e.reason)
|
| 76 |
+
)
|
| 77 |
+
if adjudications:
|
| 78 |
+
final = max(adjudications, key=lambda e: e.decided_at).decision.value
|
| 79 |
+
elif not reviews:
|
| 80 |
+
final = "not screened"
|
| 81 |
+
else:
|
| 82 |
+
decisions = {e.decision.value for e in reviews.values()}
|
| 83 |
+
if len(decisions) > 1:
|
| 84 |
+
final = "conflict"
|
| 85 |
+
elif len(reviews) < reviewers_required:
|
| 86 |
+
final = "awaiting second reviewer"
|
| 87 |
+
else:
|
| 88 |
+
final = decisions.pop()
|
| 89 |
+
return per_reviewer, final, reasons
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@router.get("/records.csv")
|
| 93 |
+
async def export_records(protocol_id: str) -> Response:
|
| 94 |
+
protocol = await _protocol_or_404(protocol_id)
|
| 95 |
+
run = await storage.get_search_run(protocol_id)
|
| 96 |
+
if run is None:
|
| 97 |
+
raise HTTPException(status_code=404, detail="No search has been run yet")
|
| 98 |
+
screening = await storage.list_screening_records(protocol_id)
|
| 99 |
+
required = protocol.reviewers_required
|
| 100 |
+
|
| 101 |
+
header = [
|
| 102 |
+
"paper_id", "title", "authors", "journal", "year", "doi", "url", "found_in", "abstract",
|
| 103 |
+
"title_abstract_by_reviewer", "title_abstract_final", "title_abstract_reasons",
|
| 104 |
+
"full_text_by_reviewer", "full_text_final", "full_text_reasons",
|
| 105 |
+
]
|
| 106 |
+
rows = []
|
| 107 |
+
for p in run.results:
|
| 108 |
+
mine = [e for e in screening if e.paper_id == p.id]
|
| 109 |
+
ta = _stage_summary([e for e in mine if e.stage == ScreeningStage.TITLE_ABSTRACT], required)
|
| 110 |
+
ft = _stage_summary([e for e in mine if e.stage == ScreeningStage.FULL_TEXT], required)
|
| 111 |
+
rows.append(
|
| 112 |
+
[p.id, p.title, p.authors, p.journal, p.year, p.doi, p.url, p.found_in, p.abstract, *ta, *ft]
|
| 113 |
+
)
|
| 114 |
+
return _csv(f"{_slug(protocol.title)}-records.csv", header, rows)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
@router.get("/screening.csv")
|
| 118 |
+
async def export_screening(protocol_id: str) -> Response:
|
| 119 |
+
protocol = await _protocol_or_404(protocol_id)
|
| 120 |
+
entries = await storage.list_screening_records(protocol_id)
|
| 121 |
+
header = [
|
| 122 |
+
"decided_at", "paper_id", "stage", "decision", "reviewer", "role",
|
| 123 |
+
"reason_category", "reason", "note",
|
| 124 |
+
]
|
| 125 |
+
rows = [
|
| 126 |
+
[e.decided_at, e.paper_id, e.stage, e.decision, e.reviewer, e.role,
|
| 127 |
+
e.reason_category, e.reason, e.note]
|
| 128 |
+
for e in entries
|
| 129 |
+
]
|
| 130 |
+
return _csv(f"{_slug(protocol.title)}-screening-audit.csv", header, rows)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
@router.get("/extraction.csv")
|
| 134 |
+
async def export_extraction(protocol_id: str) -> Response:
|
| 135 |
+
protocol = await _protocol_or_404(protocol_id)
|
| 136 |
+
entries = await storage.list_extraction_records(protocol_id)
|
| 137 |
+
run = await storage.get_search_run(protocol_id)
|
| 138 |
+
titles = {p.id: p.title for p in run.results} if run else {}
|
| 139 |
+
header = [
|
| 140 |
+
"extracted_at", "paper_id", "title", "extracted_by", "outcome", "evidence_stream",
|
| 141 |
+
"direction", "species", "route", "duration", "noael", "loael", "effect_finding",
|
| 142 |
+
"controlled_exposure", "exposure_prior_to_outcome", "individual_outcome_data",
|
| 143 |
+
"comparison_group", "notes",
|
| 144 |
+
]
|
| 145 |
+
rows = [
|
| 146 |
+
[e.extracted_at, e.paper_id, titles.get(e.paper_id), e.extracted_by, e.outcome,
|
| 147 |
+
e.evidence_stream, e.direction, e.species, e.route, e.duration, e.noael, e.loael,
|
| 148 |
+
e.effect_finding, e.controlled_exposure, e.exposure_prior_to_outcome,
|
| 149 |
+
e.individual_outcome_data, e.comparison_group, e.notes]
|
| 150 |
+
for e in entries
|
| 151 |
+
]
|
| 152 |
+
return _csv(f"{_slug(protocol.title)}-extraction.csv", header, rows)
|