| """ |
| Question-type classifier for MedQA validation cases (P1). |
| |
| Classifies USMLE-style questions by type using heuristic regex patterns |
| on the question stem. This enables type-aware scoring and stratified reporting. |
| """ |
| from __future__ import annotations |
|
|
| import re |
| from enum import Enum |
| from typing import TYPE_CHECKING |
|
|
| if TYPE_CHECKING: |
| from validation.base import ValidationCase |
|
|
|
|
| class QuestionType(str, Enum): |
| DIAGNOSTIC = "diagnostic" |
| TREATMENT = "treatment" |
| MECHANISM = "mechanism" |
| LAB_FINDING = "lab_finding" |
| PHARMACOLOGY = "pharmacology" |
| EPIDEMIOLOGY = "epidemiology" |
| ETHICS = "ethics" |
| ANATOMY = "anatomy" |
| OTHER = "other" |
|
|
|
|
| |
| _STEM_PATTERNS: list[tuple[str, QuestionType]] = [ |
| |
| (r"most likely diagnosis", QuestionType.DIAGNOSTIC), |
| (r"most likely cause", QuestionType.DIAGNOSTIC), |
| (r"most likely explanation", QuestionType.DIAGNOSTIC), |
| (r"what is the diagnosis", QuestionType.DIAGNOSTIC), |
| (r"diagnosis is", QuestionType.DIAGNOSTIC), |
| (r"most likely condition", QuestionType.DIAGNOSTIC), |
| (r"most likely has", QuestionType.DIAGNOSTIC), |
| (r"most likely suffer", QuestionType.DIAGNOSTIC), |
| (r"most likely experiencing", QuestionType.DIAGNOSTIC), |
|
|
| |
| (r"mechanism of action", QuestionType.MECHANISM), |
| (r"pathophysiology", QuestionType.MECHANISM), |
| (r"mediator.*(responsible|involved)", QuestionType.MECHANISM), |
| (r"(inhibit|block|activate).*receptor", QuestionType.MECHANISM), |
| (r"cross[\s-]?link", QuestionType.MECHANISM), |
| (r"most likely (due to|caused by|result of|secondary to)", QuestionType.MECHANISM), |
|
|
| |
| (r"drug.*(target|mechanism|receptor|inhibit)", QuestionType.PHARMACOLOGY), |
| (r"(target|act on|bind).*(receptor|enzyme|channel)", QuestionType.PHARMACOLOGY), |
| (r"mode of action", QuestionType.PHARMACOLOGY), |
|
|
| |
| (r"most likely (finding|result)", QuestionType.LAB_FINDING), |
| (r"expected (finding|result|value)", QuestionType.LAB_FINDING), |
| (r"characteristic (finding|feature|appearance)", QuestionType.LAB_FINDING), |
| (r"(agar|culture|stain|gram|biopsy).*(show|reveal|demonstrate)", QuestionType.LAB_FINDING), |
| (r"(laboratory|lab).*(result|finding|value)", QuestionType.LAB_FINDING), |
| (r"most likely (show|reveal|demonstrate)", QuestionType.LAB_FINDING), |
|
|
| |
| (r"(structure|nerve|artery|vein|muscle|ligament).*(damaged|injured|affected|involved)", QuestionType.ANATOMY), |
| (r"which.*(nerve|artery|vein|muscle|vessel)", QuestionType.ANATOMY), |
|
|
| |
| (r"(risk factor|prevalence|incidence|odds ratio|relative risk)", QuestionType.EPIDEMIOLOGY), |
| (r"most (common|frequent).*(cause|risk|complication)", QuestionType.EPIDEMIOLOGY), |
|
|
| |
| (r"most appropriate (next step|management|treatment|intervention|therapy|pharmacotherapy)", QuestionType.TREATMENT), |
| (r"best (next step|initial step|management|treatment)", QuestionType.TREATMENT), |
| (r"recommended (treatment|management|therapy)", QuestionType.TREATMENT), |
| (r"most appropriate.*(action|course)", QuestionType.TREATMENT), |
| (r"next (best )?step in (management|treatment|evaluation)", QuestionType.TREATMENT), |
| ] |
|
|
| |
| |
| _ETHICS_KEYWORDS = re.compile( |
| r"(tell|inform|disclose|report|consent|refuse|autonomy|confidentiality|" |
| r"assent|surrogate|advance directive|do not resuscitate|DNR|ethics|ethical|" |
| r"duty to warn|breach|malpractice|negligence|capacity|competence)", |
| re.IGNORECASE, |
| ) |
|
|
|
|
| def classify_question(case: "ValidationCase") -> QuestionType: |
| """ |
| Classify a MedQA question by type using heuristics on the question stem. |
| |
| Looks at metadata["question_stem"] first, falls back to |
| ground_truth["full_question"], then input_text. |
| |
| Returns: |
| QuestionType enum value |
| """ |
| stem = case.metadata.get("question_stem", "") |
| full_q = case.ground_truth.get("full_question", case.input_text) |
|
|
| |
| result = QuestionType.OTHER |
| for text in [stem, full_q]: |
| if not text: |
| continue |
| text_lower = text.lower() |
| for pattern, qtype in _STEM_PATTERNS: |
| if re.search(pattern, text_lower): |
| result = qtype |
| break |
| if result != QuestionType.OTHER: |
| break |
|
|
| |
| |
| if result == QuestionType.TREATMENT: |
| search_text = stem + " " + full_q |
| if _ETHICS_KEYWORDS.search(search_text): |
| result = QuestionType.ETHICS |
|
|
| return result |
|
|
|
|
| def classify_question_from_text(question_text: str) -> QuestionType: |
| """ |
| Classify a raw question string (no ValidationCase needed). |
| Useful for ad-hoc classification. |
| """ |
| text_lower = question_text.lower() |
| for pattern, qtype in _STEM_PATTERNS: |
| if re.search(pattern, text_lower): |
| |
| if qtype == QuestionType.TREATMENT and _ETHICS_KEYWORDS.search(question_text): |
| return QuestionType.ETHICS |
| return qtype |
| return QuestionType.OTHER |
|
|
|
|
| |
| DIAGNOSTIC_TYPES = {QuestionType.DIAGNOSTIC} |
| PIPELINE_APPROPRIATE_TYPES = { |
| QuestionType.DIAGNOSTIC, |
| QuestionType.TREATMENT, |
| QuestionType.LAB_FINDING, |
| } |
|
|