""" Uncertainty & Committee Disagreement Sampler for Clinical Active Learning. Filters large clinical corpora to select the top ~5% of ambiguous, borderline, or disagreeing clinical mentions to present to human clinicians. """ import json from pathlib import Path from typing import Any, Dict, List, Optional class UncertaintySampler: """ Computes uncertainty, margin ambiguity, and committee disagreement scores for clinical entities. """ @staticmethod def score_entity_uncertainty(entity: Dict[str, Any]) -> float: """ Computes an uncertainty score in [0.0, 1.0] where 1.0 represents highest uncertainty. Signals considered: 1. Least Confidence: GLiNER score near decision boundary (0.30 - 0.55). 2. Candidate Margin Ambiguity: Top-1 vs Top-2 candidate score difference. 3. Committee Disagreement: Span detected but unmapped to standard SCTID. 4. Category Vague-ness: Findings vs strict surgical procedures. """ top_m = entity.get("top_match", {}) conf = float(top_m.get("confidence") or top_m.get("score") or 0.85) # 1. Least confidence signal (peaks around 0.45) # Closer to 0.45 -> maximum uncertainty confidence_uncertainty = max(0.0, 1.0 - abs(conf - 0.45) * 2.0) # 2. Candidate Margin Ambiguity candidates = entity.get("candidates", []) margin_uncertainty = 0.0 if len(candidates) >= 2: s1 = float(candidates[0].get("score", 0.9)) s2 = float(candidates[1].get("score", 0.8)) diff = abs(s1 - s2) if diff < 0.10: # Tight margin -> highly ambiguous margin_uncertainty = 1.0 - (diff / 0.10) # 3. Committee Disagreement (unmapped or auto-matched) cid = str(top_m.get("concept_id", "")) disagreement_boost = 0.0 if not cid.isdigit() or cid.startswith("ACR_") or cid.startswith("SNOMED_AUTO"): disagreement_boost = 0.35 # 4. Synthesize composite score composite = ( 0.40 * confidence_uncertainty + 0.35 * margin_uncertainty + 0.25 * disagreement_boost ) return round(min(1.0, composite), 3) @classmethod def rank_entities_for_review(cls, entities: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Adds uncertainty scores and ranks entities with highest uncertainty first.""" enriched = [] for ent in entities: u_score = cls.score_entity_uncertainty(ent) ent_copy = dict(ent) ent_copy["uncertainty_score"] = u_score ent_copy["review_priority"] = "HIGH" if u_score >= 0.60 else "MEDIUM" if u_score >= 0.35 else "LOW" enriched.append(ent_copy) enriched.sort(key=lambda x: x["uncertainty_score"], reverse=True) return enriched @classmethod def build_sample_queue(cls) -> List[Dict[str, Any]]: """ Generates a curated queue of candidate edge cases from clinical notes that need clinician verification. """ base_dir = Path(__file__).resolve().parent.parent.parent notes_file = base_dir / "web" / "challenge_notes.json" queue_items = [] if notes_file.exists(): try: with open(notes_file, "r", encoding="utf-8") as f: notes = json.load(f) # Pick representative cases for nid, text in list(notes.items())[:3]: queue_items.append({ "queue_id": f"queue_{nid}", "note_id": nid, "title": f"Discharge Note {nid}", "text_snippet": text[:400] + "...", "full_text": text, "status": "PENDING_REVIEW", "priority": "HIGH", "reason": "Borderline laterality and complex diagnostic phrases detected", }) except Exception: pass if not queue_items: queue_items.append({ "queue_id": "queue_default_01", "note_id": "10043750-DS-6", "title": "Discharge Summary: Colon Cancer & Colectomy", "text_snippet": "Patient presented with a diagnosis of right colon cancer...", "full_text": "Patient presented with a diagnosis of right colon cancer diagnosed after colonoscopy.", "status": "PENDING_REVIEW", "priority": "HIGH", "reason": "Laterality qualifier and modifier resolution verification", }) return queue_items