Spaces:
Running
Running
| """ | |
| Independent dual-grader review of already-inserted discharge-letter cases. | |
| Loads stored letters + model outputs, assigns each case to exactly one of | |
| Grader 1 / Grader 2, and saves grades without mixing the two raters. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import tempfile | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| import streamlit as st | |
| from dotenv import load_dotenv | |
| APP_DIR = Path(__file__).resolve().parent | |
| def _detect_repo_dir() -> Path: | |
| here = APP_DIR | |
| for candidate in (here, here.parent): | |
| if (candidate / "feedback_data.json").exists(): | |
| return candidate | |
| if (candidate / "app.py").exists() and (candidate / "backend.py").exists(): | |
| return candidate | |
| if here.name in {"pages", "grading_space"}: | |
| return here.parent | |
| return here | |
| REPO_DIR = _detect_repo_dir() | |
| load_dotenv(REPO_DIR / ".env") | |
| HF_DATASET_REPO = "Vrda/im-error-check-data" | |
| HF_CASES_FILE = "feedback_data.json" | |
| HF_GRADES_FILE = "independent_grades.json" | |
| HF_STATE_FILE = "grader_state.json" | |
| LOCAL_GRADES_FILE = REPO_DIR / "independent_grades.json" | |
| LOCAL_STATE_FILE = REPO_DIR / "grader_state.json" | |
| LOCAL_FEEDBACK_FILE = REPO_DIR / "feedback_data.json" | |
| def _case_id_map_path() -> Path: | |
| candidates = [ | |
| APP_DIR / "case_id_map.json", | |
| REPO_DIR / "grading_space" / "case_id_map.json", | |
| REPO_DIR / "pages" / "case_id_map.json", | |
| ] | |
| for path in candidates: | |
| if path.exists(): | |
| return path | |
| return candidates[0] | |
| CASE_ID_MAP_FILE = _case_id_map_path() | |
| GRADER_CHOICES = { | |
| "Grader 1": "grader_1", | |
| "Grader 2": "grader_2", | |
| } | |
| GRADER_LABELS = {value: key for key, value in GRADER_CHOICES.items()} | |
| ERROR_JUDGMENT_OPTIONS = ["Correct", "Overcalled", "Unclear"] | |
| USEFULNESS_OPTIONS = ["Not useful", "Somewhat", "Very useful"] | |
| REVIEW_HELP_OPTIONS = ["Not at all", "Somewhat", "A lot"] | |
| HARM_OPTIONS = ["No", "Possibly", "Yes"] | |
| PREFERRED_MODEL_OPTIONS = [ | |
| "DeepSeek Reasoner", | |
| "GPT-OSS-120B", | |
| "About the same", | |
| "Neither helpful", | |
| ] | |
| SECTION_HEADERS = [ | |
| "Prezime i ime", | |
| "MBOO", | |
| "Adresa", | |
| "Datum dolaska", | |
| "Datum rođenja", | |
| "Datum otpusta", | |
| "Trijažna kategorija", | |
| "Dijagnoze", | |
| "Ostale dijagnoze", | |
| "Podaci s trijaže", | |
| "Vitalni znakovi", | |
| "Razlog dolaska", | |
| "Anamneza", | |
| "Status", | |
| "Dijagnostičke i konzilijarne pretrage", | |
| "Terapija", | |
| "Zaključak", | |
| "Uputa izabranom", | |
| "Ostale preporuke", | |
| "Address", | |
| "Date of arrival", | |
| "Arrival date", | |
| "Date of birth", | |
| "Date of discharge", | |
| "Discharge date", | |
| "Triage category", | |
| "Diagnoses", | |
| "Triage data", | |
| "Vital signs", | |
| "Reason for arrival", | |
| "Anamnesis", | |
| "Therapy", | |
| "Conclusion", | |
| "Instructions for", | |
| "Other recommendations", | |
| ] | |
| # ------------------------------------------------------------------------- | |
| # Hub / local persistence | |
| # ------------------------------------------------------------------------- | |
| def _hf_token() -> str | None: | |
| return os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") | |
| def _download_hub_json(filename: str): | |
| token = _hf_token() | |
| if not token: | |
| return None | |
| try: | |
| from huggingface_hub import HfApi | |
| api = HfApi(token=token) | |
| local_path = api.hf_hub_download( | |
| repo_id=HF_DATASET_REPO, | |
| filename=filename, | |
| repo_type="dataset", | |
| ) | |
| with open(local_path, "r", encoding="utf-8") as handle: | |
| return json.load(handle) | |
| except Exception: | |
| return None | |
| def _upload_hub_json(filename: str, data, message: str) -> str | None: | |
| token = _hf_token() | |
| if not token: | |
| return "No HF token — saved locally only." | |
| try: | |
| from huggingface_hub import HfApi | |
| api = HfApi(token=token) | |
| with tempfile.NamedTemporaryFile( | |
| mode="w", suffix=".json", delete=False, encoding="utf-8" | |
| ) as tmp: | |
| json.dump(data, tmp, ensure_ascii=False, indent=2) | |
| tmp_path = tmp.name | |
| api.upload_file( | |
| path_or_fileobj=tmp_path, | |
| path_in_repo=filename, | |
| repo_id=HF_DATASET_REPO, | |
| repo_type="dataset", | |
| commit_message=message, | |
| ) | |
| os.unlink(tmp_path) | |
| return None | |
| except Exception as exc: | |
| return str(exc) | |
| def _read_local_json(path: Path, default): | |
| if not path.exists(): | |
| return default | |
| with open(path, "r", encoding="utf-8") as handle: | |
| return json.load(handle) | |
| def _write_local_json(path: Path, data): | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| with open(path, "w", encoding="utf-8") as handle: | |
| json.dump(data, handle, ensure_ascii=False, indent=2) | |
| def load_grades() -> list: | |
| hub = _download_hub_json(HF_GRADES_FILE) | |
| local = _read_local_json(LOCAL_GRADES_FILE, []) | |
| if isinstance(hub, list) and len(hub) >= len(local): | |
| return hub | |
| return local if isinstance(local, list) else [] | |
| def load_state() -> dict: | |
| hub = _download_hub_json(HF_STATE_FILE) | |
| local = _read_local_json(LOCAL_STATE_FILE, {"assignments": {}}) | |
| if not isinstance(local, dict): | |
| local = {"assignments": {}} | |
| local.setdefault("assignments", {}) | |
| if isinstance(hub, dict) and isinstance(hub.get("assignments"), dict): | |
| if len(hub["assignments"]) >= len(local["assignments"]): | |
| return hub | |
| return local | |
| def save_grades(grades: list) -> str | None: | |
| _write_local_json(LOCAL_GRADES_FILE, grades) | |
| return _upload_hub_json( | |
| HF_GRADES_FILE, grades, f"independent grade #{len(grades)}" | |
| ) | |
| def save_state(state: dict) -> str | None: | |
| _write_local_json(LOCAL_STATE_FILE, state) | |
| return _upload_hub_json(HF_STATE_FILE, state, "grader assignment update") | |
| # ------------------------------------------------------------------------- | |
| # Case loading | |
| # ------------------------------------------------------------------------- | |
| def load_case_id_map() -> dict[str, str]: | |
| mapping = _read_local_json(CASE_ID_MAP_FILE, []) | |
| return {item["timestamp"]: item["case_id"] for item in mapping} | |
| def _case_id_for(entry: dict, index: int, ts_map: dict[str, str]) -> str: | |
| return ts_map.get(entry.get("timestamp")) or f"case_{index:03d}" | |
| def load_cases() -> list[dict]: | |
| raw = _download_hub_json(HF_CASES_FILE) | |
| if not isinstance(raw, list) or not raw: | |
| raw = _read_local_json(LOCAL_FEEDBACK_FILE, []) | |
| ts_map = load_case_id_map() | |
| cases = [] | |
| for index, entry in enumerate(raw, 1): | |
| if not isinstance(entry, dict): | |
| continue | |
| cases.append( | |
| { | |
| "case_id": _case_id_for(entry, index, ts_map), | |
| "timestamp": entry.get("timestamp", ""), | |
| "clinical_input": entry.get("clinical_input") or "", | |
| "translation": entry.get("translation") or "", | |
| "medication_lookup_context": entry.get("medication_lookup_context") or "", | |
| "model_a_output": entry.get("model_a_output") or "", | |
| "model_b_output": entry.get("model_b_output") or "", | |
| "model_a_latency": entry.get("model_a_latency"), | |
| "model_b_latency": entry.get("model_b_latency"), | |
| "both_models_complete": bool(entry.get("both_models_complete")), | |
| } | |
| ) | |
| cases.sort(key=lambda item: item["case_id"]) | |
| return cases | |
| # ------------------------------------------------------------------------- | |
| # Assignment helpers | |
| # ------------------------------------------------------------------------- | |
| def graded_case_ids(grades: list, grader_id: str) -> set[str]: | |
| return { | |
| item.get("case_id") | |
| for item in grades | |
| if item.get("grader_id") == grader_id and item.get("case_id") | |
| } | |
| def assigned_to(state: dict, case_id: str) -> str | None: | |
| record = state.get("assignments", {}).get(case_id) or {} | |
| return record.get("grader_id") | |
| def available_case_ids(cases: list, state: dict, grades: list, grader_id: str) -> list[str]: | |
| mine = graded_case_ids(grades, grader_id) | |
| available = [] | |
| for case in cases: | |
| case_id = case["case_id"] | |
| owner = assigned_to(state, case_id) | |
| if owner and owner != grader_id: | |
| continue | |
| if case_id in mine: | |
| continue | |
| available.append(case_id) | |
| return available | |
| def checkout_next_case(cases: list, grader_id: str) -> tuple[str | None, str | None]: | |
| state = load_state() | |
| grades = load_grades() | |
| remaining = available_case_ids(cases, state, grades, grader_id) | |
| if not remaining: | |
| return None, "No unassigned cases left for this grader." | |
| case_id = remaining[0] | |
| owner = assigned_to(state, case_id) | |
| if owner and owner != grader_id: | |
| return None, f"{case_id} was just taken by the other grader. Try again." | |
| state.setdefault("assignments", {})[case_id] = { | |
| "grader_id": grader_id, | |
| "assigned_at": datetime.now(timezone.utc).isoformat(), | |
| "status": "in_progress", | |
| } | |
| warning = save_state(state) | |
| return case_id, warning | |
| def release_case(case_id: str, grader_id: str) -> str | None: | |
| state = load_state() | |
| record = state.get("assignments", {}).get(case_id) | |
| if not record: | |
| return None | |
| if record.get("grader_id") != grader_id: | |
| return "This case belongs to the other grader." | |
| if record.get("status") == "completed": | |
| return "Completed cases cannot be released." | |
| state["assignments"].pop(case_id, None) | |
| return save_state(state) | |
| def mark_case_completed(case_id: str, grader_id: str) -> str | None: | |
| state = load_state() | |
| state.setdefault("assignments", {})[case_id] = { | |
| "grader_id": grader_id, | |
| "assigned_at": (state.get("assignments", {}).get(case_id) or {}).get( | |
| "assigned_at", datetime.now(timezone.utc).isoformat() | |
| ), | |
| "completed_at": datetime.now(timezone.utc).isoformat(), | |
| "status": "completed", | |
| } | |
| return save_state(state) | |
| # ------------------------------------------------------------------------- | |
| # Display helpers | |
| # ------------------------------------------------------------------------- | |
| def parse_model_json(raw: str) -> dict: | |
| text = (raw or "").strip() | |
| if text.startswith("```"): | |
| first_newline = text.index("\n") | |
| last_fence = text.rfind("```") | |
| text = text[first_newline + 1 : last_fence].strip() | |
| try: | |
| return json.loads(text) | |
| except json.JSONDecodeError: | |
| start = text.find("{") | |
| end = text.rfind("}") + 1 | |
| if start != -1 and end > start: | |
| return json.loads(text[start:end]) | |
| raise | |
| def parse_model_result(raw: str) -> dict: | |
| if not (raw or "").strip(): | |
| return { | |
| "success": False, | |
| "error_message": "No stored output for this model.", | |
| "errors": [], | |
| "suggestions": [], | |
| "summary": {}, | |
| } | |
| try: | |
| parsed = parse_model_json(raw) | |
| except Exception as exc: | |
| return { | |
| "success": False, | |
| "error_message": f"Could not parse stored output: {exc}", | |
| "errors": [], | |
| "suggestions": [], | |
| "summary": {}, | |
| } | |
| summary = parsed.get("summary") | |
| if not isinstance(summary, dict): | |
| summary = {"overall_assessment": summary or ""} | |
| return { | |
| "success": True, | |
| "errors": parsed.get("errors") or [], | |
| "suggestions": parsed.get("suggestions") or [], | |
| "summary": summary, | |
| } | |
| def format_letter(text: str) -> str: | |
| if not text.strip(): | |
| return "_No discharge letter stored for this case._" | |
| cleaned = text.replace("\r\n", "\n").replace("\r", "\n") | |
| cleaned = re.sub(r"[ \t]+\n", "\n", cleaned) | |
| cleaned = re.sub(r"\n{3,}", "\n\n", cleaned).strip() | |
| header_pattern = re.compile( | |
| r"^(" + "|".join(re.escape(item) for item in SECTION_HEADERS) + r")\b", | |
| re.IGNORECASE, | |
| ) | |
| lines = [] | |
| for line in cleaned.split("\n"): | |
| stripped = line.strip() | |
| if stripped and header_pattern.match(stripped): | |
| lines.append(f"\n### {stripped}\n") | |
| else: | |
| lines.append(stripped) | |
| return "\n".join(lines).strip() | |
| def extract_diagnosis(text: str) -> str: | |
| match = re.search( | |
| r"(?:Dijagnoze|Diagnoses)\s*\n+([^\n]+)", | |
| text or "", | |
| flags=re.IGNORECASE, | |
| ) | |
| if match: | |
| return match.group(1).strip() | |
| return "Diagnosis not listed" | |
| def extract_arrival(text: str) -> str: | |
| match = re.search( | |
| r"(?:Datum dolaska|Date of arrival|Arrival date):\s*([^\n]+)", | |
| text or "", | |
| flags=re.IGNORECASE, | |
| ) | |
| if match: | |
| return match.group(1).strip() | |
| return "" | |
| SEVERITY_LABELS = { | |
| "critical": "🔴 Critical", | |
| "high": "🟠 High", | |
| "medium": "🟡 Medium", | |
| "low": "🟢 Low", | |
| } | |
| CATEGORY_LABELS = { | |
| "medication_error": "Medication", | |
| "diagnostic_error": "Diagnostic", | |
| "dosing_error": "Dosing", | |
| "documentation_error": "Documentation", | |
| "lab_interpretation_error": "Lab Interpretation", | |
| "contraindication": "Contraindication", | |
| "omission": "Omission", | |
| "disposition_safety": "Disposition Safety", | |
| "other": "Other", | |
| "documentation_quality": "Documentation Quality", | |
| "clinical_workflow": "Clinical Workflow", | |
| "patient_safety": "Patient Safety", | |
| "continuity_of_care": "Continuity of Care", | |
| "patient_education": "Patient Education", | |
| "followup_quality": "Follow-up Quality", | |
| "medication_safety": "Medication Safety", | |
| "completeness": "Completeness", | |
| } | |
| PRIORITY_LABELS = {"high": "🔴 High", "medium": "🟡 Medium", "low": "🟢 Low"} | |
| SAFETY_RISK_LABELS = {"low": "🟢 Low", "moderate": "🟡 Moderate", "high": "🔴 High"} | |
| DOC_QUALITY_LABELS = { | |
| "excellent": "Excellent", | |
| "good": "Good", | |
| "adequate": "Adequate", | |
| "needs_improvement": "Needs Improvement", | |
| "poor": "Poor", | |
| } | |
| def render_model_output(result: dict, latency): | |
| if not result["success"]: | |
| st.error(result["error_message"]) | |
| return | |
| if latency is not None: | |
| st.caption(f"Original response time: {latency}s") | |
| summary = result["summary"] | |
| if summary.get("overall_assessment"): | |
| parts = [f"**Summary:** {summary['overall_assessment']}"] | |
| if summary.get("safety_risk"): | |
| parts.append( | |
| "**Safety risk:** " | |
| + SAFETY_RISK_LABELS.get(summary["safety_risk"], summary["safety_risk"]) | |
| ) | |
| if summary.get("documentation_quality"): | |
| parts.append( | |
| "**Documentation quality:** " | |
| + DOC_QUALITY_LABELS.get( | |
| summary["documentation_quality"], summary["documentation_quality"] | |
| ) | |
| ) | |
| st.markdown(" | ".join(parts)) | |
| if result["errors"]: | |
| for index, err in enumerate(result["errors"], 1): | |
| sev = SEVERITY_LABELS.get(err.get("severity", ""), err.get("severity", "")) | |
| cat = CATEGORY_LABELS.get(err.get("category", ""), err.get("category", "")) | |
| quote = ( | |
| f'<br><em>Quote: "{err.get("quote")}"</em>' if err.get("quote") else "" | |
| ) | |
| why = "" | |
| if err.get("why_it_matters"): | |
| why = f"<br><strong>Why it matters:</strong> {err['why_it_matters']}" | |
| st.markdown( | |
| f'<div class="error-card">' | |
| f"<strong>Error {index}</strong> — {sev} " | |
| f'<span class="category-badge">{cat}</span><br>' | |
| f"{err.get('description', '')}" | |
| f"{quote}{why}</div>", | |
| unsafe_allow_html=True, | |
| ) | |
| else: | |
| st.info("No errors identified.") | |
| if result["suggestions"]: | |
| for index, sug in enumerate(result["suggestions"], 1): | |
| cat = CATEGORY_LABELS.get(sug.get("category", ""), sug.get("category", "")) | |
| priority = "" | |
| if sug.get("priority"): | |
| pri = PRIORITY_LABELS.get(sug["priority"], sug["priority"]) | |
| priority = f' <span class="category-badge">{pri}</span>' | |
| details = "" | |
| if sug.get("quote"): | |
| details += f'<br><em>Original: "{sug["quote"]}"</em>' | |
| if sug.get("suggested_rewrite"): | |
| details += ( | |
| "<br><strong>Suggested rewrite:</strong> " | |
| f"{sug['suggested_rewrite']}" | |
| ) | |
| if sug.get("why_recommended"): | |
| details += ( | |
| "<br><strong>Why recommended:</strong> " | |
| f"{sug['why_recommended']}" | |
| ) | |
| st.markdown( | |
| f'<div class="suggestion-card">' | |
| f"<strong>Suggestion {index}</strong> " | |
| f'<span class="category-badge">{cat}</span>{priority}<br>' | |
| f"{sug.get('description', '')}" | |
| f"{details}</div>", | |
| unsafe_allow_html=True, | |
| ) | |
| def progress_counts(cases: list, state: dict, grades: list): | |
| total = len(cases) | |
| g1 = sum( | |
| 1 | |
| for item in state.get("assignments", {}).values() | |
| if item.get("grader_id") == "grader_1" | |
| ) | |
| g2 = sum( | |
| 1 | |
| for item in state.get("assignments", {}).values() | |
| if item.get("grader_id") == "grader_2" | |
| ) | |
| done1 = len(graded_case_ids(grades, "grader_1")) | |
| done2 = len(graded_case_ids(grades, "grader_2")) | |
| unassigned = total - g1 - g2 | |
| return total, done1, done2, g1, g2, max(unassigned, 0) | |
| # ------------------------------------------------------------------------- | |
| # Page | |
| # ------------------------------------------------------------------------- | |
| try: | |
| st.set_page_config(page_title="IM Independent Grading", page_icon="🩺", layout="wide") | |
| except Exception: | |
| pass | |
| st.markdown( | |
| """ | |
| <style> | |
| .error-card { | |
| background: #2d1f1f; border-left: 4px solid #e53e3e; | |
| border-radius: 8px; padding: 0.8rem 1rem; margin: 0.5rem 0; | |
| color: #fde8e8; | |
| } | |
| .error-card strong { color: #feb2b2; } | |
| .error-card em { color: #fbd38d; } | |
| .suggestion-card { | |
| background: #1a2e1a; border-left: 4px solid #38a169; | |
| border-radius: 8px; padding: 0.8rem 1rem; margin: 0.5rem 0; | |
| color: #c6f6d5; | |
| } | |
| .suggestion-card strong { color: #9ae6b4; } | |
| .model-header-a { | |
| background: #1a2332; border-left: 4px solid #63b3ed; | |
| border-radius: 8px; padding: 0.6rem 1rem; margin-bottom: 0.5rem; | |
| } | |
| .model-header-b { | |
| background: #2d1f3d; border-left: 4px solid #b794f4; | |
| border-radius: 8px; padding: 0.6rem 1rem; margin-bottom: 0.5rem; | |
| } | |
| .category-badge { | |
| display: inline-block; background: #2d3748; color: #e2e8f0; | |
| padding: 2px 8px; border-radius: 12px; font-size: 0.8em; margin-right: 4px; | |
| } | |
| </style> | |
| """, | |
| unsafe_allow_html=True, | |
| ) | |
| for key, default in [ | |
| ("grader_id", ""), | |
| ("grader_ready", False), | |
| ("current_case_id", ""), | |
| ("status_message", ""), | |
| ]: | |
| if key not in st.session_state: | |
| st.session_state[key] = default | |
| access_pin = os.environ.get("ACCESS_PIN") or os.environ.get("GRADER_ACCESS_PIN") | |
| if access_pin: | |
| if "access_ok" not in st.session_state: | |
| st.session_state.access_ok = False | |
| if not st.session_state.access_ok: | |
| st.info("This grading page is PIN-protected.") | |
| entered_access = st.text_input("Access PIN", type="password", key="access_pin") | |
| if st.button("Unlock"): | |
| if entered_access == access_pin: | |
| st.session_state.access_ok = True | |
| st.rerun() | |
| st.error("Incorrect access PIN.") | |
| st.stop() | |
| def cached_cases() -> list[dict]: | |
| return load_cases() | |
| cases = cached_cases() | |
| cases_by_id = {item["case_id"]: item for item in cases} | |
| st.title("🩺 Independent Grading — Discharge Letter Error-Check") | |
| st.markdown( | |
| "Review **already inserted** cases. The letter and both model outputs are " | |
| "preloaded. Each case is assigned to **exactly one** grader." | |
| ) | |
| st.warning( | |
| "⚠️ **RESEARCH TOOL**: AI-generated findings require physician verification. " | |
| "Do not use as sole basis for clinical decisions." | |
| ) | |
| with st.sidebar: | |
| st.header("Grader identity") | |
| selected_label = st.radio( | |
| "Who is grading?", | |
| list(GRADER_CHOICES.keys()), | |
| index=0 if st.session_state.grader_id != "grader_2" else 1, | |
| ) | |
| selected_id = GRADER_CHOICES[selected_label] | |
| expected_pin = os.environ.get(f"{selected_id.upper()}_PIN") or os.environ.get( | |
| f"GRADER_{selected_id[-1]}_PIN" | |
| ) | |
| entered_pin = "" | |
| if expected_pin: | |
| entered_pin = st.text_input("PIN", type="password", key="grader_pin") | |
| if st.button("Start as this grader", type="primary"): | |
| if expected_pin and entered_pin != expected_pin: | |
| st.error("Incorrect PIN for this grader.") | |
| else: | |
| if st.session_state.grader_id != selected_id: | |
| st.session_state.current_case_id = "" | |
| st.session_state.grader_id = selected_id | |
| st.session_state.grader_ready = True | |
| st.rerun() | |
| if st.session_state.grader_ready: | |
| st.success(f"Signed in as **{GRADER_LABELS[st.session_state.grader_id]}**") | |
| if st.button("Switch grader"): | |
| st.session_state.grader_ready = False | |
| st.session_state.grader_id = "" | |
| st.session_state.current_case_id = "" | |
| st.rerun() | |
| st.markdown("---") | |
| st.header("Progress") | |
| if cases: | |
| state_now = load_state() | |
| grades_now = load_grades() | |
| total, done1, done2, assigned1, assigned2, unassigned = progress_counts( | |
| cases, state_now, grades_now | |
| ) | |
| st.metric("Cases in pool", total) | |
| st.metric("Grader 1 completed", f"{done1} / assigned {assigned1}") | |
| st.metric("Grader 2 completed", f"{done2} / assigned {assigned2}") | |
| st.metric("Still unassigned", unassigned) | |
| st.caption("A case checked out by one grader never appears for the other.") | |
| else: | |
| st.info("No stored cases found yet.") | |
| if not st.session_state.grader_ready: | |
| st.info("Select **Grader 1** or **Grader 2** in the sidebar, then start.") | |
| st.stop() | |
| if not cases: | |
| st.error( | |
| "Could not load stored cases. Check HF_TOKEN / HUGGINGFACE_TOKEN and " | |
| "that `Vrda/im-error-check-data` contains feedback_data.json." | |
| ) | |
| st.stop() | |
| grader_id = st.session_state.grader_id | |
| state = load_state() | |
| grades = load_grades() | |
| remaining = available_case_ids(cases, state, grades, grader_id) | |
| if st.session_state.status_message: | |
| st.info(st.session_state.status_message) | |
| st.session_state.status_message = "" | |
| if not st.session_state.current_case_id: | |
| st.subheader(f"{GRADER_LABELS[grader_id]} queue") | |
| st.write(f"**{len(remaining)}** case(s) still available to you.") | |
| col_next, col_refresh = st.columns([2, 1]) | |
| with col_next: | |
| if st.button("Check out next case", type="primary", disabled=not remaining): | |
| case_id, warning = checkout_next_case(cases, grader_id) | |
| if case_id: | |
| st.session_state.current_case_id = case_id | |
| if warning: | |
| st.session_state.status_message = f"Saved locally. Hub sync: {warning}" | |
| st.rerun() | |
| else: | |
| st.warning(warning or "No case available.") | |
| with col_refresh: | |
| if st.button("Refresh queue"): | |
| st.cache_data.clear() | |
| st.rerun() | |
| if not remaining: | |
| st.success("You have no remaining cases. Thank you.") | |
| st.stop() | |
| case = cases_by_id.get(st.session_state.current_case_id) | |
| if case is None: | |
| st.session_state.current_case_id = "" | |
| st.warning("That case is no longer in the pool. Returning to the queue.") | |
| st.rerun() | |
| owner = assigned_to(state, case["case_id"]) | |
| if owner and owner != grader_id: | |
| st.session_state.current_case_id = "" | |
| st.warning("This case belongs to the other grader.") | |
| st.rerun() | |
| diagnosis = extract_diagnosis(case["clinical_input"] or case["translation"]) | |
| arrival = extract_arrival(case["clinical_input"] or case["translation"]) | |
| st.subheader(f"{case['case_id']} — {diagnosis}") | |
| meta = [f"Inserted {case['timestamp'][:10]}"] | |
| if arrival: | |
| meta.append(f"Arrival {arrival}") | |
| st.caption(" · ".join(meta)) | |
| action_l, action_r = st.columns(2) | |
| with action_l: | |
| if st.button("Release this case back to the pool"): | |
| warning = release_case(case["case_id"], grader_id) | |
| st.session_state.current_case_id = "" | |
| st.session_state.status_message = warning or "Case released." | |
| st.rerun() | |
| with action_r: | |
| if st.button("Back to queue (keep assignment)"): | |
| st.session_state.current_case_id = "" | |
| st.rerun() | |
| st.markdown("---") | |
| st.header("Discharge letter") | |
| st.markdown(format_letter(case["clinical_input"])) | |
| if case["translation"]: | |
| with st.expander("English translation"): | |
| st.markdown(format_letter(case["translation"])) | |
| if case["medication_lookup_context"]: | |
| with st.expander("Medication lookup context"): | |
| st.code(case["medication_lookup_context"]) | |
| st.markdown("---") | |
| st.header("Model comparison") | |
| result_a = parse_model_result(case["model_a_output"]) | |
| result_b = parse_model_result(case["model_b_output"]) | |
| col_a, col_b = st.columns(2, gap="large") | |
| with col_a: | |
| st.markdown( | |
| '<div class="model-header-a"><h4 style="color:#3182ce; margin:0">' | |
| "DeepSeek Reasoner</h4></div>", | |
| unsafe_allow_html=True, | |
| ) | |
| render_model_output(result_a, case.get("model_a_latency")) | |
| with col_b: | |
| st.markdown( | |
| '<div class="model-header-b"><h4 style="color:#805ad5; margin:0">' | |
| "GPT-OSS-120B</h4></div>", | |
| unsafe_allow_html=True, | |
| ) | |
| render_model_output(result_b, case.get("model_b_latency")) | |
| st.markdown("---") | |
| st.subheader("Physician Feedback (Research)") | |
| st.markdown( | |
| "*Same form as the original study app. Your grades are stored under your grader ID.*" | |
| ) | |
| feedback_data = {} | |
| available_models = [ | |
| ("model_a", "DeepSeek Reasoner", result_a), | |
| ("model_b", "GPT-OSS-120B", result_b), | |
| ] | |
| case_key = case["case_id"] | |
| for model_key, model_label, res in available_models: | |
| st.markdown(f"#### {model_label}") | |
| error_ratings = [] | |
| if res["success"] and res["errors"]: | |
| st.markdown("**For each flagged error:**") | |
| for index, err in enumerate(res["errors"]): | |
| description = err.get("description", "") | |
| st.markdown( | |
| f"*Error {index + 1}:* {description[:160]}" | |
| f"{'...' if len(description) > 160 else ''}" | |
| ) | |
| judgment = st.radio( | |
| "Assessment", | |
| ERROR_JUDGMENT_OPTIONS, | |
| horizontal=True, | |
| key=f"{case_key}_{model_key}_err_{index}_judgment", | |
| label_visibility="collapsed", | |
| ) | |
| error_ratings.append( | |
| { | |
| "error_text": description, | |
| "model_category": err.get("category", ""), | |
| "model_severity": err.get("severity", ""), | |
| "judgment": judgment.lower(), | |
| } | |
| ) | |
| elif res["success"]: | |
| st.info("No errors flagged — rate the suggestions and overall output below.") | |
| else: | |
| st.caption("This model has no usable stored output.") | |
| model_feedback = {"errors": error_ratings} | |
| if res["success"] and res["suggestions"]: | |
| model_feedback["suggestions_useful"] = st.radio( | |
| "**Were the suggestions clinically useful?**", | |
| USEFULNESS_OPTIONS, | |
| horizontal=True, | |
| key=f"{case_key}_{model_key}_sug_useful", | |
| ) | |
| if res["success"]: | |
| model_feedback["helpful_for_review"] = st.radio( | |
| "**Would this output help you review this discharge letter?**", | |
| REVIEW_HELP_OPTIONS, | |
| horizontal=True, | |
| key=f"{case_key}_{model_key}_review_help", | |
| ) | |
| model_feedback["harm_if_unchecked"] = st.radio( | |
| "**If a junior colleague acted on these findings without your review, " | |
| "could it cause harm?**", | |
| HARM_OPTIONS, | |
| horizontal=True, | |
| key=f"{case_key}_{model_key}_harm", | |
| ) | |
| feedback_data[model_key] = model_feedback | |
| st.markdown("---") | |
| st.markdown("#### This case") | |
| preferred_model = st.radio( | |
| "**Which model was more useful for this case?**", | |
| PREFERRED_MODEL_OPTIONS, | |
| horizontal=True, | |
| key=f"{case_key}_preferred_model", | |
| ) | |
| missed_errors_important = st.radio( | |
| "**Did either model miss an important error?**", | |
| ["No", "Yes"], | |
| horizontal=True, | |
| key=f"{case_key}_missed_errors_important", | |
| ) | |
| missed_errors = "" | |
| if missed_errors_important == "Yes": | |
| missed_errors = st.text_area( | |
| "Describe what was missed:", | |
| placeholder="e.g. Both models missed inadequate anticoagulation counseling...", | |
| key=f"{case_key}_missed_errors", | |
| height=80, | |
| ) | |
| comments = st.text_area( | |
| "Additional comments (optional):", | |
| placeholder="Any other observations about the models' performance?", | |
| key=f"{case_key}_fb_comments", | |
| height=80, | |
| ) | |
| if st.button("Submit grade", type="primary"): | |
| if missed_errors_important == "Yes" and not missed_errors.strip(): | |
| st.warning("Please describe the missed error(s), or change the answer to No.") | |
| else: | |
| latest_grades = load_grades() | |
| already = any( | |
| item.get("case_id") == case["case_id"] | |
| and item.get("grader_id") == grader_id | |
| for item in latest_grades | |
| ) | |
| if already: | |
| st.warning("You already graded this case. It was not overwritten.") | |
| else: | |
| entry = { | |
| "case_id": case["case_id"], | |
| "source_timestamp": case["timestamp"], | |
| "grader_id": grader_id, | |
| "grader_label": GRADER_LABELS[grader_id], | |
| "graded_at": datetime.now(timezone.utc).isoformat(), | |
| "feedback_schema_version": 2, | |
| "ratings": feedback_data, | |
| "preferred_model": preferred_model, | |
| "missed_errors_important": missed_errors_important == "Yes", | |
| "missed_errors": missed_errors.strip(), | |
| "comments": comments, | |
| } | |
| latest_grades.append(entry) | |
| grade_warning = save_grades(latest_grades) | |
| state_warning = mark_case_completed(case["case_id"], grader_id) | |
| st.session_state.current_case_id = "" | |
| notes = [] | |
| if grade_warning: | |
| notes.append(f"grades hub: {grade_warning}") | |
| if state_warning: | |
| notes.append(f"assignment hub: {state_warning}") | |
| st.session_state.status_message = ( | |
| f"Saved {case['case_id']} as {GRADER_LABELS[grader_id]}. " | |
| + (" ".join(notes) if notes else "Synced to local JSON and Hub.") | |
| ) | |
| st.rerun() | |
| st.markdown("---") | |
| st.caption( | |
| "Independent grading | Internal Medicine Error-Check | Prospective Research Study 2026" | |
| ) | |