diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..0104b1b46b02ac1f5cba4a932b6f9f301fe13bec --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +.git +.venv +**/node_modules +**/__pycache__ +**/.pytest_cache +**/.ruff_cache +**/.mypy_cache +ui/dist +data/synthea/output +data/multicare/cache +data/models +*.duckdb +*.parquet +*.gguf +*.onnx +*.safetensors +.env +.env.local +.hf_token +scratch/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..c6fee4ae96816ee83d689ae7f32bee16d4d2418d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + lint-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Set up Python + run: uv python install 3.11 + - name: Install deps (no ML extras) + run: uv sync --extra dev + - name: Ruff + run: uv run ruff check . + - name: Ruff format check + run: uv run ruff format --check . + - name: Pytest + run: uv run pytest -m "not ml" --cov=app --cov=eval_harness --cov-report=term-missing + + ui-build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ui + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/package-lock.json + - run: npm ci + - run: npm run build + - run: npm run lint diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a58094e284358fb01742c5a8a1f82c8e1e8fc9da --- /dev/null +++ b/.gitignore @@ -0,0 +1,64 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +.venv/ +venv/ +env/ +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ +dist/ +build/ + +# uv +uv.lock + +# Node +node_modules/ +ui/dist/ +ui/.vite/ +*.log + +# Data — models, generated datasets, large downloads stay local +data/synthea/output/ +data/multicare/cache/ +data/models/ +*.parquet +*.duckdb +*.sqlite +*.gguf +*.onnx +*.safetensors +*.bin +*.pt + +# But keep the small staged CSVs / reference tables +!data/reference/** + +# Env + secrets +.env +.env.local +.hf_token +*.pem + +# OS +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ +*.swp + +# Local scratch +scratch/ +*.scratch.* + +# Docker +.docker/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff239865f705bd3f172257029e9c88a676218263 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.4 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + args: [--maxkb=2048] + - id: check-merge-conflict diff --git a/AGENT_BRIEF.md b/AGENT_BRIEF.md new file mode 100644 index 0000000000000000000000000000000000000000..90b30a77861ecbcfaf2ed16e999c427875aa6779 --- /dev/null +++ b/AGENT_BRIEF.md @@ -0,0 +1,381 @@ +# Agent Brief — HCC Risk-Adjustment Coder with MEAT Justification + +> **You are taking over end-to-end ownership of this project.** This brief is self-contained: it explains who the work is for, why it exists, the full scope, the data, the architecture, the UX, the evaluation harness, and the deliverables. Read it straight through before planning. The human (Charles) expects you to operate autonomously on execution but will approve scope-level and design-level decisions. Flag trade-offs; do not silently resolve them. + +--- + +## 1. Project identity + +- **Name (working):** `hcc-meat-coder` +- **One-line pitch:** Upload a clinical note → get suggested HCC codes (CMS-HCC v28) with each code backed by quoted **MEAT** evidence (Monitor / Evaluate / Assess / Treat), calibrated confidence, and an over-coding risk flag. +- **Deployment target:** HuggingFace Space (Gradio), plus a model-card repo and an eval-harness repo. +- **Tier:** Flagship (Tier 1) — one of two anchor projects in a 2026 healthtech portfolio. + +--- + +## 2. Portfolio context (why this project exists) + +The person commissioning this is **Charles**, a builder with: +- Background in **basic medicinal science** + ML (LLMs, time series, CV, DL, biomedical informatics). +- Shipped a **CP-SAT doctor-rostering web app** with real users — the "I ship production tools" signal already exists. +- Building a HuggingFace portfolio aimed at **2026 healthtech employers** (Abridge, Ambience, Tempus, Hippocratic AI, Commure, FinThrive, Latent, Cohere Health, CodaMetrix, etc.), constrained to **public data + a single laptop + HF Spaces** as the demo surface. + +**Why this project in particular.** Autonomous medical coding is one of the hottest 2025/2026 healthtech buckets (~$450M into RCM/coding/billing in 2025; Smarter Technologies, Commure, FinThrive, CodaMetrix all hiring). **HCC (Hierarchical Condition Categories)** coding specifically drives CMS risk-adjusted payments for Medicare Advantage — a multi-billion-dollar line item where under-coding loses money and over-coding triggers RADV audits. **MEAT** is the CMS-expected documentation standard for any HCC claim. The industry has near-zero credible HuggingFace demos in this niche. A well-executed Space here is the kind of artifact a coding-automation startup's recruiter will screenshot. + +**Portfolio positioning alongside Charles's other planned flagship (1.3 Polypharmacy / DDI Agent):** +- 1.2 (this) proves **clinical-NLP + structured-output + calibration** chops against a real payer framework. +- 1.3 proves the **medicinal-science moat** (pharmacology reasoning). +- Both reuse shared infra: a Gradio component library, a citation-faithfulness verifier, a subgroup-sliced metrics widget, a "dataset card + disclaimer" template. Coordinate with the 1.3 stream so the shared infra is built once. + +**Signals this project must send to hiring managers:** +1. You understand the CMS-HCC v28 model and the RADV-audit context (not just ICD-10 mapping). +2. You build calibrated, auditable outputs — not raw LLM prose. +3. You know the difference between "the model thinks this is a diagnosis" and "this diagnosis meets MEAT criteria for the DOS." +4. You can ship a UI a non-technical reviewer (or a coder) can operate in 30 seconds. + +--- + +## 3. Mission & success criteria + +**Mission.** Ship a public HuggingFace Space that, given a clinical note (synthetic or public), produces: +1. A ranked list of suggested HCC codes (CMS-HCC v28) with the underlying ICD-10 codes. +2. For each suggested HCC, a **MEAT justification** block: one or more **quoted spans** from the note for each of Monitor, Evaluate, Assess, Treat — with explicit "insufficient MEAT" flagging when a category has no supporting evidence. +3. A **calibrated confidence** score per code with a visible reliability diagram on a held-out set. +4. An **over-coding risk flag** per code (heuristic: weak MEAT coverage + low-confidence NER + high-RAF-weight HCC → elevated audit risk). +5. Subgroup-sliced metrics (by age band, note length, specialty) visible in the Space. + +**Success criteria — in priority order.** +- **S1:** Every claim in the output is traceable to a quoted span in the input (no fabrications). Verified by an NLI-style faithfulness check in the eval harness. +- **S2:** The Space runs on the free CPU tier of HF Spaces with < 8s latency per note on the preloaded gallery. +- **S3:** Subgroup-sliced metrics are shown in the Space itself, not hidden in a notebook. +- **S4:** A recruiter skim (30s) makes the clinical value obvious without technical jargon. +- **S5:** Reasonable absolute quality on a held-out set: macro-F1 > 0.55 on HCC code suggestion is a strong target given the public-data constraint. **Do not chase SOTA; chase interpretability + auditability.** + +--- + +## 4. Industry context an agent should internalize + +- CMS-HCC v28 (phased in 2024–2026) replaces v24; it has ~115 payment HCCs with different **RAF weights**. Accurate HCC capture is what Medicare Advantage plans pay RCM vendors for. +- **MEAT** is the standard documentation framework for supporting a diagnosis on a date of service: Monitor (signs/symptoms tracked), Evaluate (test/exam results reviewed), Assess (clinician's impression documented), Treat (medication, referral, therapy). Not all four are required, but the stronger the MEAT coverage, the lower the RADV-audit risk. +- **RADV (Risk Adjustment Data Validation)** audits by CMS recover overpayments if documentation doesn't support submitted HCCs. This is the pain an HCC coder automates against. +- Competitors / reference points: **CodaMetrix**, **Smarter Technologies**, **RapidClaims**, **Fathom**, **Nym Health**, **Regard**, **Iodine Software**. None of them have public HF demos. +- Vocabulary to use fluently in README / model card / blog: **RAF score, HCC capture, suspect list, chart review, retrospective vs prospective coding, MEAT-compliance, RADV-defensibility, specificity coding**. + +--- + +## 5. Prior art on HuggingFace (what to beat) + +Research pass from April 2026 found: +- OpenMed collection (481 NER models, 380+ fine-grained) — strong at entity spans, no HCC/MEAT framing. +- `blaze999/Medical-NER`, `d4data/biomedical-ner-all`, `samrawal/bert-base-uncased_clinical-ner` — similar. +- `obi/Medical Note Deidentification` (~39 likes). +- No polished HCC / MEAT / risk-adjustment Space exists. + +**Differentiation.** Charles's project is not another medical NER demo. It is a **coding-workflow artifact** with four distinctive features the existing landscape does not deliver: (1) HCC crosswalk, (2) MEAT-quoted evidence, (3) calibration + over-coding risk, (4) RADV-framed UX copy. + +--- + +## 6. Scope + +### In scope +- Synthetic or public clinical notes as input (free-text). +- Medical NER, ICD-10 linking, ICD-10 → HCC crosswalk (v28), MEAT classification/extraction, calibration, over-coding risk heuristic. +- Gradio Space with three-panel UI. +- Eval harness: held-out macro-F1, per-HCC precision/recall, MEAT coverage stats, citation-faithfulness NLI, calibration curve, subgroup slices. +- Model card documenting data licenses, model limits, failure modes. +- One blog post (technical + clinical-workflow framed). + +### Out of scope (call out explicitly in README) +- Claim submission, 837/EDI generation, or any direct payer integration. +- Any use on real PHI. +- Any production clinical use — disclaimer banner required on the Space. +- ICD-10 PCS (procedure) coding. This is diagnosis coding only. +- E/M level coding (CPT). + +--- + +## 7. Data sources + +### 7.1 Notes (model input for training and demo) + +| Source | License | Use | +|---|---|---| +| **Synthea** (Apache 2.0, generates FHIR R4 + CSV + notes via `--exporter.text.export=true`) | Fully redistributable. | Primary training + demo notes. Generate ~10k patients with diverse comorbidities. | +| **MultiCaRe** (93k PMC clinical cases + 130k images, openly curated, 2025) | Permissive per the CC license per paper — check per case. | Secondary corpus for real-prose case-style notes. Great for the demo gallery (publishable). | +| **MIMIC-IV-Note** | **PhysioNet credentialed — DO NOT use for training a model distributed on HF, DO NOT redistribute.** | Only acceptable path: credentialed evaluation off-HF to quote eval numbers in the README. Default: skip entirely. | + +**Default data posture:** Synthea for everything; MultiCaRe for gallery examples; skip MIMIC to keep redistribution clean. + +### 7.2 Code sets and crosswalks (all public) + +- **ICD-10-CM** (CMS / CDC): annual release, public-domain code list and descriptions. +- **CMS-HCC v28** model: public on the CMS Risk Adjustment page. Includes HCC list, ICD-10-to-HCC crosswalk, and RAF coefficients by segment (Community / Institutional / New Enrollee). +- **HCC hierarchy** (trumping rules): embedded in the v28 release. +- **RxNorm** (NLM, public) — optional, for medication mentions supporting Treat. +- **LOINC** (permissive) — optional, for lab mentions supporting Evaluate/Monitor. + +### 7.3 MEAT training data + +There is no public "MEAT-annotated" corpus. Approach: +1. **Synthetic weak labels**: generate Synthea notes with a template engine that inserts known MEAT-shaped sentences per condition, giving you perfect labels at scale for a distillation-style training set. +2. **LLM-generated gold**: prompt a large open LLM to annotate 500–2000 real-prose MultiCaRe cases for MEAT spans per diagnosis; manually spot-check a 200-case subset; publish the resulting dataset on HF as `chaaarlieyap/meat-annotated-cases` with methodology transparent in the card. +3. **Active-learning loop**: use the fine-tuned MEAT model to flag low-confidence cases, re-annotate those. + +### 7.4 Evaluation data + +- Held-out Synthea set with known ground-truth diagnoses (built-in). +- Held-out MultiCaRe set with the LLM-gold MEAT labels. +- Optional: participate in **n2c2** or **CLEF eHealth** shared tasks for external validation numbers to quote (credentialed — quote metrics only). + +--- + +## 8. Architecture & components + +High-level pipeline (each arrow is a pluggable component with its own tests): + +``` + Note (free text) + ├─► [A] Section Segmenter ──► sections (HPI, ROS, PE, A/P, Meds, Labs, Plan) + ├─► [B] Clinical NER ──► spans: Conditions / Meds / Labs / Procedures / Findings + ├─► [C] Negation + Temporality classifier ──► polarity, time (historical vs current) + ├─► [D] ICD-10 Linker ──► (span, ICD-10 code, linker_conf) + ├─► [E] HCC Crosswalk ──► (ICD-10 → HCC v28, RAF weight, hierarchy class) + ├─► [F] MEAT Extractor ──► for each HCC candidate, spans labeled M/E/A/T + ├─► [G] Calibrator ──► per-HCC calibrated probability (isotonic) + ├─► [H] Over-coding Risk ──► heuristic score combining MEAT coverage, NER conf, RAF weight, hierarchy trumping + └─► [I] Verifier / Faithfulness check ──► asserts every quoted MEAT span exists verbatim in source note +``` + +### Component A — Section segmenter +- Input: raw note. Output: dict of sections. +- Implementation: rule-based header matcher (SOAP, H&P templates) + small fallback classifier. Synthea notes have predictable structure; MultiCaRe prose requires the fallback. + +### Component B — Clinical NER +- **Starter model:** `blaze999/Medical-NER` or one of the OpenMed NER models as a baseline. +- **Upgrade path:** fine-tune `microsoft/BiomedBERT` or `emilyalsentzer/Bio_ClinicalBERT` (if accessible) on Synthea labels + n2c2-derived weak labels. If a model gives acceptable out-of-the-box performance, don't fine-tune for its own sake. +- Output: `BIO`-tagged spans with type ∈ {Condition, Medication, LabTest, Procedure, Finding, Anatomy}. + +### Component C — Negation + temporality +- Use **NegSpacy** / **medspaCy** or a small BERT classifier fine-tuned on 2010 i2b2-derived synthetic labels. +- Output: `(span, negated: bool, temporality: {current, historical, hypothetical, family_history})`. +- **Critical:** filter out negated and family-history mentions before HCC attribution. This is a major real-world failure mode of naive coders. + +### Component D — ICD-10 linker +- Approach: bi-encoder retrieval (SapBERT or MedCPT embeddings) over the ICD-10-CM description corpus, followed by cross-encoder re-ranking. +- Output: top-k ICD-10 codes per Condition span with linker confidence. + +### Component E — HCC crosswalk +- Pure lookup: ICD-10 → HCC v28 (table join). +- Apply **HCC hierarchy trumping** rules: within a disease hierarchy, only the most severe HCC counts (e.g., HCC 19 Diabetes w/ complications trumps HCC 21 Diabetes w/o). +- Output per candidate: `(HCC, RAF_weight, hierarchy_group, trumped: bool)`. + +### Component F — MEAT extractor +- **Small LLM with structured output**. Candidates: `microsoft/Phi-4-mini`, `Qwen/Qwen2.5-3B-Instruct`, or `google/medgemma-4b-it` if latency allows on CPU. Prompt the model with (a) the note's Assessment/Plan section, (b) the candidate HCC, (c) a JSON schema for MEAT quotes. +- Use **constrained decoding** (Outlines, XGrammar, or llama.cpp grammar) to guarantee JSON validity. +- **Hard requirement:** every quote must be an exact substring of the source note. Enforce with a post-decode check — if the model hallucinates a quote, drop it and mark that MEAT category "Insufficient." + +### Component G — Calibrator +- Per-HCC (or per-HCC-frequency-bucket) **isotonic regression** on held-out Synthea. Publish the reliability diagram in the Space. +- If per-HCC is too sparse, calibrate by HCC-frequency bucket (frequent / medium / rare). + +### Component H — Over-coding risk heuristic +- Score components (transparent, not a black box): + - MEAT coverage: `count of filled MEAT categories / 4` (weight 0.4). + - NER confidence on the supporting Condition span (weight 0.2). + - Linker confidence on the ICD-10 match (weight 0.2). + - RAF weight of the HCC: higher weight → higher audit risk if weakly supported (weight 0.1). + - Hierarchy trumping: does another HCC in the same hierarchy also appear with stronger evidence? (weight 0.1). +- Output: `{"risk": "low" | "medium" | "high", "reasons": [...]}`. Surface reasons in the UI. + +### Component I — Verifier / faithfulness +- Enforces the substring guarantee (post-decode check above), and separately runs a lightweight NLI model (`cross-encoder/nli-deberta-v3-base` or a clinical-NLI variant) to check that the quoted MEAT span actually entails the MEAT category. +- Counts of verifier-filtered spans are logged per run and shown in the Space's metrics panel. + +### Shared infra (coordinate with 1.3 stream) +- `gradio_components/`: EvidenceHighlighter, SubgroupSliceMetrics, CitationFaithfulnessPanel, DatasetCardWithDisclaimer. +- `eval_harness/faithfulness.py`: NLI-based claim-to-source verifier. Reused by 1.3. + +--- + +## 9. UX / HuggingFace Space design + +### Layout (Gradio `Blocks`) + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ Banner: ⚠ Research demo. Not for clinical use. Synthetic data only. │ +├────────────────────────┬──────────────────────────────┬──────────────────┤ +│ LEFT — Note panel │ CENTER — HCC suggestions │ RIGHT — Drill-in │ +│ (editable / gallery) │ (ranked list) │ (click an HCC) │ +│ │ │ │ +│ [Load example ▼] │ HCC 19 Diabetes w/ complic. │ MEAT breakdown: │ +│ [paste or edit note] │ RAF 0.302 │ M: "A1c 9.2..." │ +│ │ Conf: 0.81 [calibrated] │ E: "fundo exam.."│ +│ Spans highlighted │ Risk: LOW │ A: "DM with PN.." │ +│ by component: │ │ T: "insulin .." │ +│ ■ Condition │ HCC 85 CHF │ │ +│ ■ Medication │ RAF 0.331 │ Over-coding risk:│ +│ ■ Lab │ Conf: 0.58 │ LOW (3 of 4 │ +│ │ Risk: MEDIUM │ MEAT filled, │ +│ │ ✗ E missing │ strong NER) │ +│ │ │ │ +└────────────────────────┴──────────────────────────────┴──────────────────┘ + [Metrics tab] [Calibration tab] [Subgroups tab] [About / Model card] +``` + +### Example gallery (preload 8–12 cases) +Each gallery item should be a **deliberately different** challenge: +1. Straightforward DM2 with complications — easy positive case. +2. CHF with preserved EF — tests specificity coding. +3. Family history of malignancy (must NOT code) — tests negation/family-history filter. +4. "History of" chronic condition — tests temporality (historical → may still be active HCC depending on persistence). +5. Ruled-out diagnosis ("r/o sepsis, not present") — tests negation. +6. Trumping case: DM w/o complications mentioned early, DM w/ neuropathy mentioned later — tests hierarchy trumping. +7. Rare HCC (e.g., HCC 17 Diabetes mellitus due to underlying condition). +8. Short vs long note — tests subgroup stability. +9. Multi-specialty note (cardiology + endocrine + nephrology) — tests multi-HCC capture. +10. Insufficient-MEAT case: diagnosis mentioned in A/P but no Monitor or Evaluate — should flag medium/high risk. + +### Copy & tone +- Write UX copy in coder vocabulary, not ML vocabulary. E.g., "Suggested capture", "MEAT compliance", "RADV-defensibility", "suspect list", "hierarchy trumping", "calibrated probability". +- Every code card has a tooltip linking to the CMS-HCC v28 public page. +- Include a visible "This is a research demo. Synthetic data. Not for clinical or billing use." banner. + +### Tabs beyond the main view +- **Metrics tab:** held-out macro-F1, per-HCC P/R/F1 bar chart. +- **Calibration tab:** reliability diagram + Brier score. +- **Subgroups tab:** sliced metrics by age band, note length bucket, specialty. +- **About tab:** data licenses, model card summary, known failure modes, feedback link. + +--- + +## 10. Evaluation harness (separate repo, linked from Space) + +Deliver as a Python package `hcc_meat_eval/` with a CLI and a notebook. + +### Metrics +- **Coding quality:** macro-F1 on HCC code suggestion, per-HCC P/R/F1, mean RAF captured vs ground-truth RAF. +- **MEAT quality:** span-level exact-match F1 vs gold spans; per-MEAT-category P/R. +- **Faithfulness:** % of quoted spans that are exact substrings of the source; NLI-verified entailment rate; hallucination rate (definitionally 0 after verifier, but count pre-verifier drops). +- **Calibration:** reliability diagram, Brier score, ECE. +- **Robustness:** performance under note-length perturbation, synonym substitution (e.g., "sugar diabetes" / "DM2" / "Type 2 diabetes mellitus"), paraphrase of A/P section. +- **Subgroup slicing:** by age band, specialty, note length, rare-HCC vs common-HCC. +- **Over-coding audit:** for each "LOW-risk" prediction, does MEAT actually support? Use held-out labels to estimate audit false-positive rate. + +### Failure-mode taxonomy +Publish a catalog of documented failure cases in the eval repo: (a) family-history mis-attribution, (b) negation miss, (c) historical-only diagnosis coded as current, (d) trumping violation, (e) fabricated MEAT quote (should be 0 after verifier), (f) MEAT quote present but semantically wrong category. Each entry links to a test case. + +--- + +## 11. Deliverables + +1. **HuggingFace Space** (`hcc-meat-coder`) — Gradio app, runs on CPU free tier. +2. **HF model repo** (`hcc-meat-coder-models`) — trained component weights with model cards. +3. **HF dataset repo** (`meat-annotated-synthea`) — the synthetic MEAT corpus you generate. Card documents generation methodology and LLM prompts used. +4. **Evaluation harness repo on GitHub** — reproducible; pinned versions; CLI for running metrics on a new note corpus. +5. **Blog post** — clinical-workflow framing first, ML details second. Title suggestion: _"Building an HCC coder that shows its work"_. Open with the coder's pain (RADV audits), not the model architecture. +6. **Short demo video** (60–90s screen recording) embedded in the Space and blog. +7. **Model card** — covers intended use, out-of-scope, data, metrics, subgroup slices, failure modes, license of each upstream component. + +--- + +## 11.5. Agent tooling / Claude Code skills setup + +Before doing any work that touches HuggingFace (creating the Space, pushing model weights, uploading datasets, editing the model card), install the HuggingFace skills plugin so you have first-class `hf` CLI capability from inside Claude Code: + +``` +claude +/plugin marketplace add huggingface/skills +/plugin install hf-cli@huggingface/skills +``` + +After install, authenticate with `hf auth login` using a write-scoped token. Use the `hf-cli` skill for: creating the Space / model / dataset repos, uploading files, editing README frontmatter, and managing Space secrets. Do **not** shell out to ad-hoc curl calls against the HF API when the skill covers the operation. + +--- + +## 12. Tech stack + +- Python 3.11, `uv` for env management. +- `gradio` ≥ 4.x for the Space, `spaces.GPU` decorator if you need burst GPU. +- `transformers`, `sentence-transformers`, `outlines` (structured output), `medspacy` or `negspacy`. +- `duckdb` for fast local joins on ICD-10 / HCC tables. +- `scikit-learn` for calibration + metrics; `matplotlib` for reliability diagrams. +- `pytest` + `ruff` + `pre-commit`. Keep CI simple (GitHub Actions). +- A small CLI via `typer` for the eval harness. + +--- + +## 13. Milestones & sequencing (target: 6 weeks) + +**Week 1 — Data + crosswalks.** Set up Synthea generator, extract free-text notes, pull public ICD-10 + CMS-HCC v28 tables, build duckdb schema. Spot-check crosswalk correctness on 30 known cases. + +**Week 2 — Core NLP pipeline.** Components A–E wired end-to-end with placeholders for F and G. First crude "ICD-10 candidates from a note" prototype. + +**Week 3 — MEAT extractor (component F).** Generate synthetic MEAT training set from Synthea templates; distill a structured-output prompt; verifier enforces substring. First end-to-end "note → HCC + MEAT" output. + +**Week 4 — Calibration + over-coding risk.** Held-out isotonic; reliability diagram; risk heuristic with transparent reasons. + +**Week 5 — Space UI + gallery.** Ship Gradio Blocks layout, preload 10 example cases, write clinical-coder-voiced copy, add Metrics/Calibration/Subgroups tabs. + +**Week 6 — Eval harness, model card, blog, demo video.** Publish failure-mode catalog; write the blog with the clinical-workflow opening; record the video. + +**Stretch:** publish the synthetic MEAT dataset with a methodology card; add an "upload your own note" mode with a PHI-scanner warning. + +--- + +## 14. Risks & mitigations + +| Risk | Mitigation | +|---|---| +| LLM hallucinates MEAT quotes. | Hard substring check + NLI verifier; drop quotes that fail; mark MEAT category "Insufficient." | +| Synthea notes are too stylistically uniform → model overfits. | Mix MultiCaRe prose for training; measure subgroup metrics by note source. | +| ICD-10 → HCC v28 crosswalk errors. | Unit-test against 50 published CMS examples; publish the crosswalk version used in the model card. | +| Trumping logic is subtle. | Implement hierarchy rules from the official CMS file directly; write explicit tests. | +| CPU latency > 8s per note on free-tier Space. | Cache ICD-10 embeddings; quantize the MEAT LLM to INT8 or GGUF; precompute gallery outputs. | +| DrugBank / proprietary vocabularies pulled in by accident. | This project doesn't need DrugBank. Keep it strictly CMS / CDC / NLM sources to stay redistribution-safe. | +| Real-world HCC coders complain the demo oversimplifies. | Pre-empt with a "Known limitations" section in the model card; link to CMS documentation. | + +--- + +## 15. Style & quality bar + +- **Calibrated + auditable > peak F1.** If a choice trades 2 points of F1 for interpretability, take the interpretability. +- **Every number on screen should be reproducible** from the harness. +- **Write for a coder, not an ML researcher.** The README opens with "What this does for you if you're a risk-adjustment coder." +- **Disclaimers are always on, always visible, never ironic.** + +--- + +## 16. Handoff questions to resolve with Charles before starting + +Flag these on day one; do not silently resolve: +1. GPU budget for training: is the plan CPU-only throughout, or is there a one-off GPU run acceptable for the MEAT extractor training? +2. Scope of MultiCaRe use — are we comfortable publishing the LLM-generated MEAT labels publicly, or keep that dataset private? +3. Is a "bring your own note" upload mode in scope for v1, or only the preloaded gallery? (Has implications for the PHI warning and log retention.) +4. Target coding specificity: just the ~115 payment HCCs, or the full HCC set (incl. non-payment HCCs)? + +--- + +## 17. References + +### CMS / regulatory +- CMS Risk Adjustment — HCC v28 model files and documentation (public on cms.gov). +- CMS MLN "Evaluation and Management Services" guide — for MEAT framing. +- RADV audit methodology — CMS Final Rule 2023. + +### Datasets +- [Synthea](https://synthea.mitre.org/downloads) — Apache 2.0 +- [MultiCaRe](https://www.mdpi.com/2306-5729/10/8/123) — open clinical case dataset + +### Models (starting points) +- `blaze999/Medical-NER`, `d4data/biomedical-ner-all`, OpenMed NER collection +- `microsoft/Phi-4-mini`, `Qwen/Qwen2.5-3B-Instruct`, `google/medgemma-4b-it` +- `cross-encoder/nli-deberta-v3-base` for faithfulness NLI + +### Landscape / industry +- HuggingFace medical landscape survey (April 2026) — see parent `../PORTFOLIO_IDEAS.md` for citations. +- Rock Health 2025 year-end overview; Menlo Ventures State of AI in Healthcare 2025. +- MIT Sloan Field Guide to Deploying AI Agents in Clinical Practice. + +### Parent portfolio document +- [`../PORTFOLIO_IDEAS.md`](../PORTFOLIO_IDEAS.md) — the portfolio-level context that motivated this project. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bf208fdd28f16c047c01055574e14b9ca216445c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# Multi-stage build for the HuggingFace Docker Space. +# Stage 1: build the React UI. +# Stage 2: python runtime serving FastAPI + the compiled static UI. + +FROM node:20-alpine AS ui-builder +WORKDIR /ui +COPY ui/package.json ui/package-lock.json* ./ +RUN npm install --no-audit --no-fund +COPY ui/ ./ +RUN npm run build + + +FROM python:3.11-slim AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PORT=7860 + +# System deps: libgomp for onnxruntime / sklearn; build tools are not needed at runtime. +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Python deps. We install the base set only; `[ml]` extras are installed on +# demand via build args (the CPU free tier doesn't need the full stack for the +# rule-based baseline). +COPY pyproject.toml README.md ./ +RUN pip install --upgrade pip && \ + pip install \ + fastapi>=0.115 "uvicorn[standard]>=0.32" pydantic>=2.9 \ + duckdb>=1.1 pandas>=2.2 numpy>=2.0 scikit-learn>=1.5 \ + typer>=0.12 rich>=13.9 httpx>=0.27 + +COPY app ./app +COPY eval_harness ./eval_harness +COPY data/reference ./data/reference +COPY tests/fixtures/regression_suite.json ./tests/fixtures/regression_suite.json +COPY --from=ui-builder /ui/dist ./ui/dist + +EXPOSE 7860 +HEALTHCHECK --interval=30s --timeout=3s --retries=3 \ + CMD python -c "import httpx,sys; sys.exit(0 if httpx.get('http://127.0.0.1:7860/api/healthz').is_success else 1)" + +CMD ["python", "-m", "uvicorn", "app.api.server:app", "--host", "0.0.0.0", "--port", "7860"] diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9771dacdf48a0c63fe0a5bae2fc446f9996bffd1 --- /dev/null +++ b/README.md @@ -0,0 +1,118 @@ +# HCC-MEAT Coder + +A research demo that suggests CMS-HCC v28 codes from a clinical note with every +suggestion backed by **MEAT** (Monitor / Evaluate / Assess / Treat) evidence +quoted verbatim from the source. Built for a 2026 healthtech portfolio. + +> ⚠ Research demo. Not for clinical, billing, or coding use. Synthetic data only. + +**What it does for a risk-adjustment coder.** Drop a note in the left pane, click +**Code note**, and get: +- A ranked list of candidate HCCs with their underlying ICD-10 codes and RAF weights. +- Per-HCC MEAT evidence — each MEAT category either quotes a verbatim span from the + note or is flagged **Insufficient**. +- A calibrated confidence score and an **over-coding (RADV) risk** flag with + transparent reasons. +- Hierarchy trumping applied per the CMS-HCC v28 disease hierarchy. + +## Quickstart + +```bash +# 1. Clone and enter +cd 1.2_HCC_Coder_MEAT + +# 2. Install Python env (uv manages 3.11 and deps) +uv sync --extra dev +# Optional: ML extras for the real Transformers-based NER / NLI +uv sync --extra dev --extra ml + +# 3. Install UI deps +(cd ui && npm install) + +# 4. Run the full stack locally +uv run hcc-meat serve # FastAPI on :7860 +(cd ui && npm run dev) # Vite dev server on :5173 (proxies /api to 7860) +``` + +Open http://localhost:5173. + +### Run the pipeline on one note (CLI, no UI) + +```bash +echo "HPI: 67yo with DM2 on metformin. A1c 9.2." > /tmp/note.txt +uv run hcc-meat code /tmp/note.txt +``` + +### Run the regression suite + faithfulness check + +```bash +uv run pytest -m "not ml" +uv run hcc-meat-eval regression +uv run hcc-meat-eval faithfulness +``` + +## How it works (Components A–I) + +``` + Note (free text) + A Section Segmenter → sections (HPI, A/P, Labs, ...) + B Clinical NER → spans: Condition / Medication / Lab / ... + C Negation/Temporality → negation + family-history + historical filter + D ICD-10 Linker → (span, ICD-10, linker_conf) + E HCC Crosswalk (v28) → HCC candidate + RAF weight + hierarchy rank + F MEAT Extractor → per HCC: list[MEATEvidence] + I Verifier → substring-check + NLI entailment check + G Calibrator → calibrated per-HCC probability + H Over-coding Risk → low/medium/high + transparent reasons + Ranked CodingResult (+ verifier drop count, latency, model refs) +``` + +Component details are in [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) and +[`AGENT_BRIEF.md`](AGENT_BRIEF.md). + +## Directory layout + +``` +app/ FastAPI backend + pipeline package + api/server.py /api/code, /api/gallery, /api/healthz + pipeline/ Components A..I, orchestrator, types + cli.py `hcc-meat code|gallery|serve` +ui/ React + Vite + Tailwind frontend + src/App.tsx Three-panel layout (Note / HCCs / Drill-in) +eval_harness/ Regression + faithfulness + (stub) metrics CLIs +data/reference/ Public ICD-10 + CMS-HCC v28 crosswalks (seed; see PROVENANCE.md) +tests/ Unit + fixture tests; regression suite +docs/ BUILD_LOG, ARCHITECTURE, EVALUATION, MODEL_CARD +Dockerfile HF Docker Space image (Node UI build + Python runtime) +``` + +## What's in this v0.1.0 scaffold + +Fully working today (with rule-based baselines in place of ML models): +- Pipeline runs end-to-end on any note string. +- Deterministic components (crosswalk E, trumping, risk heuristic H, verifier I) + implemented and unit-tested. +- 10-case regression fixture covering positive, negation, family-history, + temporality, trumping, insufficient-MEAT, multi-specialty, synonym scenarios. +- FastAPI backend + React UI with three-panel layout, disclaimer banner, gallery. +- Dockerfile for HF Docker Space. + +Placeholder / stubbed (flagged in code): +- CMS-HCC v28 RAF weights are **PLACEHOLDER** — hydrate via + `scripts/fetch_cms_hcc_v28.py` (stub). +- Components B (NER), D (linker), F (MEAT extractor) use dictionary / regex + baselines — swap for the Week-2 ML models. +- Calibrator (G) is identity until a held-out set exists to fit on. +- Subgroup slicing and calibration reliability diagram in the eval harness are stubs. + +## Disclaimers + +- No PHI. Uses Synthea (Apache 2.0) and MultiCaRe (permissive) only. +- Not a clinical decision-support tool; suggestions must not be submitted + to any payer. +- Every MEAT quote is a byte-exact substring of the input note; fabricated + quotes are dropped by Component I and counted in `verifier_drops`. + +See [`AGENT_BRIEF.md`](AGENT_BRIEF.md) for the full project brief, +[`docs/BUILD_LOG.md`](docs/BUILD_LOG.md) for the running record of what's been +built, and [`docs/EVALUATION.md`](docs/EVALUATION.md) for how to evaluate it. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b605db38dcdc1b1d634decf484cc446d0dcd0f0c --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,3 @@ +"""HCC-MEAT Coder: CMS-HCC v28 code suggester with MEAT-justified evidence.""" + +__version__ = "0.1.0" diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/api/server.py b/app/api/server.py new file mode 100644 index 0000000000000000000000000000000000000000..98ea4d062fda2aba52415ddc796ef7cfda4faefe --- /dev/null +++ b/app/api/server.py @@ -0,0 +1,121 @@ +"""FastAPI backend for the HCC-MEAT Coder Space. + +Endpoints: + POST /api/code — submit a note, receive ranked HCC candidates + MEAT evidence + GET /api/gallery — return the preloaded demo gallery (note + precomputed result) + GET /api/healthz — simple health probe + GET /api/reference/hcc — static HCC label table for tooltip content in the UI + +The app keeps a single `Pipeline` instance in state and reuses it across requests. +""" + +from __future__ import annotations + +import json +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel, Field + +from app.pipeline import Pipeline +from app.pipeline.types import CodingResult, Note + +GALLERY_PATH = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "regression_suite.json" +UI_DIST = Path(__file__).resolve().parents[2] / "ui" / "dist" + + +class CodeRequest(BaseModel): + text: str = Field(..., min_length=1, max_length=50_000) + note_id: str = Field(default="adhoc") + + +class GalleryItem(BaseModel): + case_id: str + name: str + tags: list[str] + text: str + result: CodingResult + + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.pipeline = Pipeline() + # Precompute gallery once so the UI is instant on click. + app.state.gallery = _build_gallery(app.state.pipeline) + yield + + +app = FastAPI( + title="HCC-MEAT Coder", + version="0.1.0", + description=( + "Suggests CMS-HCC v28 codes from a clinical note with MEAT-justified " + "evidence, calibrated confidence, and an over-coding risk flag." + ), + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/api/healthz") +def healthz() -> dict: + return {"status": "ok", "version": app.version} + + +@app.post("/api/code", response_model=CodingResult) +def code(req: CodeRequest) -> CodingResult: + pipeline: Pipeline = app.state.pipeline + note = Note(note_id=req.note_id, text=req.text) + return pipeline.run(note) + + +@app.get("/api/gallery", response_model=list[GalleryItem]) +def gallery() -> list[GalleryItem]: + return app.state.gallery + + +@app.get("/api/reference/hcc") +def reference_hcc() -> list[dict]: + """Return HCC labels for UI tooltips + model card linking.""" + import csv + + path = Path("data/reference/hcc_v28_labels.csv") + if not path.exists(): + raise HTTPException(500, "reference data missing") + with path.open() as f: + return list(csv.DictReader(f)) + + +def _build_gallery(pipeline: Pipeline) -> list[GalleryItem]: + if not GALLERY_PATH.exists(): + return [] + cases = json.loads(GALLERY_PATH.read_text())["cases"] + out: list[GalleryItem] = [] + for c in cases: + note = Note(note_id=c["case_id"], text=c["note_text"]) + result = pipeline.run(note) + out.append( + GalleryItem( + case_id=c["case_id"], + name=c["name"], + tags=c.get("tags", []), + text=c["note_text"], + result=result, + ) + ) + return out + + +# Mount the built React UI *last* so /api/* routes win. +if UI_DIST.exists(): + app.mount("/", StaticFiles(directory=UI_DIST, html=True), name="ui") diff --git a/app/cli.py b/app/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..31fe6f72248fb17de02f4b624938c64630820119 --- /dev/null +++ b/app/cli.py @@ -0,0 +1,95 @@ +"""`hcc-meat` command-line interface. + + hcc-meat code — run the pipeline on a single note, print JSON + hcc-meat gallery — dump the precomputed demo gallery + hcc-meat serve [--port 7860] — run the FastAPI backend with uvicorn +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import typer +from rich.console import Console +from rich.table import Table + +from app.pipeline import Pipeline +from app.pipeline.types import Note + +app = typer.Typer(help=__doc__, no_args_is_help=True) +console = Console() + + +@app.command() +def code( + note_path: Path = typer.Argument(..., help="Path to a plain-text clinical note."), + json_out: bool = typer.Option(False, "--json", help="Emit full JSON."), +) -> None: + text = note_path.read_text() + result = Pipeline().run(Note(note_id=note_path.stem, text=text)) + if json_out: + sys.stdout.write(result.model_dump_json(indent=2)) + sys.stdout.write("\n") + return + + table = Table(title=f"HCC suggestions for {note_path.name}") + table.add_column("HCC") + table.add_column("Label") + table.add_column("RAF") + table.add_column("Conf") + table.add_column("Trump", justify="center") + table.add_column("Risk") + table.add_column("MEAT (MEAT)") + for c in result.candidates: + cov = c.meat_coverage() + meat_str = "".join( + cat.value if cov[cat] else "·" for cat in list(cov.keys()) + ) + table.add_row( + c.hcc, + c.hcc_label, + f"{c.raf_weight:.3f}", + f"{c.confidence_calibrated:.2f}", + f"↓{c.hierarchy.trumped_by}" if c.hierarchy.trumped else "—", + c.risk.level.upper() if c.risk else "?", + meat_str, + ) + console.print(table) + console.print(f"[dim]verifier drops: {result.verifier_drops} | latency: {result.latency_ms:.1f} ms[/dim]") + + +@app.command() +def gallery( + fixture: Path = typer.Option( + Path("tests/fixtures/regression_suite.json"), + help="Fixture file to run against.", + ) +) -> None: + cases = json.loads(fixture.read_text())["cases"] + pipe = Pipeline() + out = [] + for c in cases: + result = pipe.run(Note(note_id=c["case_id"], text=c["note_text"])) + out.append({ + "case_id": c["case_id"], + "result": result.model_dump(), + }) + sys.stdout.write(json.dumps(out, indent=2)) + sys.stdout.write("\n") + + +@app.command() +def serve( + host: str = typer.Option("0.0.0.0", help="Bind host."), + port: int = typer.Option(7860, help="Bind port (7860 is HF Space default)."), + reload: bool = typer.Option(False, help="uvicorn --reload for dev."), +) -> None: + import uvicorn + + uvicorn.run("app.api.server:app", host=host, port=port, reload=reload) + + +if __name__ == "__main__": + app() diff --git a/app/pipeline/__init__.py b/app/pipeline/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..22dec7f41fe2d54cd7016a77b199dae703126c6b --- /dev/null +++ b/app/pipeline/__init__.py @@ -0,0 +1,30 @@ +from app.pipeline.orchestrator import Pipeline, PipelineConfig +from app.pipeline.types import ( + CodingResult, + HCCCandidate, + ICD10Link, + MEATCategory, + MEATEvidence, + Note, + NoteSection, + RiskAssessment, + Span, + SpanType, + Temporality, +) + +__all__ = [ + "Pipeline", + "PipelineConfig", + "Note", + "NoteSection", + "Span", + "SpanType", + "Temporality", + "ICD10Link", + "HCCCandidate", + "MEATCategory", + "MEATEvidence", + "RiskAssessment", + "CodingResult", +] diff --git a/app/pipeline/components/__init__.py b/app/pipeline/components/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/pipeline/components/a_segmenter.py b/app/pipeline/components/a_segmenter.py new file mode 100644 index 0000000000000000000000000000000000000000..893cc5bf3f3bd71ba054a08f74d62ee2c8f53669 --- /dev/null +++ b/app/pipeline/components/a_segmenter.py @@ -0,0 +1,81 @@ +"""Component A — Section segmenter. + +Rule-based header matcher covering the common SOAP / H&P section headers seen in +Synthea notes and typical MultiCaRe case reports. A small BERT-based fallback +classifier will be added later for notes without explicit headers. +""" + +from __future__ import annotations + +import re + +from app.pipeline.types import Note, NoteSection, SectionKind + +def _header_re(alt: str) -> re.Pattern[str]: + """Match header at start of a line, followed by either ':' (inline content ok) or end-of-line.""" + return re.compile(rf"^[ \t]*(?:{alt})[ \t]*(?::[ \t]*|$)", re.I | re.M) + + +# Ordered: more specific patterns first. +_HEADER_PATTERNS: list[tuple[SectionKind, re.Pattern[str]]] = [ + ("assessment_and_plan", _header_re(r"assessment[ \t]*(?:and|&|/)[ \t]*plan|a[ \t]*/[ \t]*p")), + ("chief_complaint", _header_re(r"chief complaint|cc")), + ("hpi", _header_re(r"history of present illness|hpi")), + ("ros", _header_re(r"review of systems|ros")), + ("physical_exam", _header_re(r"physical exam(?:ination)?|pe|exam")), + ("past_medical_history", _header_re(r"past medical history|pmh")), + ("family_history", _header_re(r"family history|fh|fhx")), + ("social_history", _header_re(r"social history|sh|shx")), + ("medications", _header_re(r"medications?|meds?|home medications?")), + ("labs", _header_re(r"labs?|laboratory(?: data)?|lab results?")), + ("assessment", _header_re(r"assessment")), + ("plan", _header_re(r"plan")), +] + + +def segment(note: Note) -> Note: + """Populate `note.sections`. Returns the same note with sections filled.""" + hits: list[tuple[int, int, SectionKind]] = [] # (header_start, header_end, kind) + for kind, pat in _HEADER_PATTERNS: + for m in pat.finditer(note.text): + hits.append((m.start(), m.end(), kind)) + + hits.sort(key=lambda h: h[0]) + + sections: list[NoteSection] = [] + if not hits: + sections = [NoteSection(kind="other", text=note.text, start=0, end=len(note.text))] + else: + # Prefix before the first header, if any. + first_start = hits[0][0] + if first_start > 0: + sections.append( + NoteSection(kind="other", text=note.text[:first_start], start=0, end=first_start) + ) + for i, (hs, he, kind) in enumerate(hits): + body_start = he + body_end = hits[i + 1][0] if i + 1 < len(hits) else len(note.text) + sections.append( + NoteSection( + kind=kind, + text=note.text[body_start:body_end], + start=body_start, + end=body_end, + ) + ) + + return note.model_copy(update={"sections": sections}) + + +def find_section(note: Note, kind: SectionKind) -> NoteSection | None: + for s in note.sections: + if s.kind == kind: + return s + return None + + +def section_for_offset(note: Note, offset: int) -> SectionKind | None: + for s in note.sections: + if s.start <= offset < s.end: + return s.kind + return None diff --git a/app/pipeline/components/b_ner.py b/app/pipeline/components/b_ner.py new file mode 100644 index 0000000000000000000000000000000000000000..2694599b3adb604f08af15b0dfe0353a7843470e --- /dev/null +++ b/app/pipeline/components/b_ner.py @@ -0,0 +1,152 @@ +"""Component B — Clinical NER. + +Interface-only with two implementations: + * `DictionaryNER`: regex dictionary matcher seeded from the crosswalk's ICD-10 + descriptions and a small medication/lab keyword list. Used for smoke-testing + and in the CI path (no ML deps). + * `TransformersNER`: wraps a HuggingFace token-classification model (e.g., + `blaze999/Medical-NER`). Imported lazily so CI without `[ml]` extras runs. + +The dictionary NER is obviously not competitive; it exists so the full pipeline +runs end-to-end from day 1 and fixture tests don't have to mock. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol + +from app.pipeline.components.a_segmenter import section_for_offset +from app.pipeline.types import Note, Span, SpanType + + +class NER(Protocol): + def __call__(self, note: Note) -> list[Span]: ... + + +# ---------------------------------------------------------------- dictionary NER + +# Seed vocabularies. Intentionally small — the real path is TransformersNER. +_MEDICATION_KEYWORDS = [ + "metformin", "insulin", "glargine", "lisinopril", "atorvastatin", "furosemide", + "warfarin", "aspirin", "albuterol", "tiotropium", "prednisone", "losartan", + "carvedilol", "spironolactone", "apixaban", "dapagliflozin", "empagliflozin", + "sitagliptin", "semaglutide", "hydrochlorothiazide", +] +_LAB_KEYWORDS = [ + "hba1c", "a1c", "creatinine", "egfr", "bnp", "pro-bnp", "troponin", "inr", + "hemoglobin", "hgb", "glucose", "potassium", "sodium", "ldl", "hdl", "ggt", + "ast", "alt", "bilirubin", +] +_PROCEDURE_KEYWORDS = [ + "dialysis", "hemodialysis", "echocardiogram", "echo", "stent", "pci", + "bronchoscopy", "fundoscopy", "fundus exam", "colonoscopy", +] +_FINDING_KEYWORDS = [ + "edema", "rales", "crackles", "wheezing", "murmur", "dyspnea", "orthopnea", + "claudication", "pruritus", "fatigue", +] + + +def _condition_terms_from_icd_descriptions(desc_file: Path) -> list[str]: + """Mine rough condition surface forms from the ICD-10 descriptions CSV. + + We take the description, lowercase it, and emit a handful of trimmed variants. + This is intentionally lossy; the real NER model replaces it. + """ + terms: set[str] = set() + if not desc_file.exists(): + return [] + with desc_file.open() as f: + next(f, None) # header + for line in f: + parts = line.rstrip("\n").split(",", 1) + if len(parts) != 2: + continue + desc = parts[1].strip().strip('"').lower() + # Short variants a clinician would actually write: + terms.add(desc) + if "diabetes mellitus" in desc: + terms.update({"diabetes", "dm", "t2dm", "t1dm", "type 2 diabetes", "type 1 diabetes"}) + if "heart failure" in desc: + terms.update({"chf", "heart failure", "congestive heart failure"}) + if "chronic obstructive pulmonary disease" in desc: + terms.update({"copd"}) + if "chronic kidney disease" in desc: + terms.update({"ckd", "chronic kidney disease"}) + if "atrial fibrillation" in desc: + terms.update({"afib", "a-fib", "atrial fibrillation"}) + if "angina" in desc: + terms.update({"angina"}) + if "cirrhosis" in desc: + terms.update({"cirrhosis"}) + if "hepatitis" in desc: + terms.update({"hepatitis", "hep b", "hep c"}) + return sorted(terms, key=len, reverse=True) # longest-first for greedy matching + + +@dataclass +class DictionaryNER: + reference_dir: Path = field(default_factory=lambda: Path("data/reference")) + _condition_terms: list[str] = field(default_factory=list, init=False) + + def __post_init__(self) -> None: + self._condition_terms = _condition_terms_from_icd_descriptions( + self.reference_dir / "icd10_cm_descriptions.csv" + ) + + def __call__(self, note: Note) -> list[Span]: + spans: list[Span] = [] + spans.extend(self._match(note, self._condition_terms, SpanType.CONDITION, conf=0.55)) + spans.extend(self._match(note, _MEDICATION_KEYWORDS, SpanType.MEDICATION, conf=0.70)) + spans.extend(self._match(note, _LAB_KEYWORDS, SpanType.LAB_TEST, conf=0.70)) + spans.extend(self._match(note, _PROCEDURE_KEYWORDS, SpanType.PROCEDURE, conf=0.65)) + spans.extend(self._match(note, _FINDING_KEYWORDS, SpanType.FINDING, conf=0.55)) + spans.sort(key=lambda s: s.start) + spans = _dedupe_overlaps(spans) + return spans + + def _match( + self, note: Note, terms: Iterable[str], span_type: SpanType, conf: float + ) -> list[Span]: + out: list[Span] = [] + text = note.text + lower = text.lower() + for term in terms: + if not term: + continue + pat = re.compile(rf"\b{re.escape(term)}\b") + for m in pat.finditer(lower): + start, end = m.start(), m.end() + out.append( + Span( + start=start, + end=end, + text=text[start:end], + span_type=span_type, + ner_confidence=conf, + section_kind=section_for_offset(note, start), + ) + ) + return out + + +def _dedupe_overlaps(spans: list[Span]) -> list[Span]: + """Keep the longest span at each overlap; break ties by higher confidence.""" + if not spans: + return spans + spans = sorted(spans, key=lambda s: (s.start, -(s.end - s.start), -s.ner_confidence)) + kept: list[Span] = [] + cursor = -1 + for s in spans: + if s.start >= cursor: + kept.append(s) + cursor = s.end + return kept + + +def default_ner() -> NER: + return DictionaryNER() diff --git a/app/pipeline/components/c_negation.py b/app/pipeline/components/c_negation.py new file mode 100644 index 0000000000000000000000000000000000000000..7a6750dfb5a075166aa8a9cadc0213ae3537199a --- /dev/null +++ b/app/pipeline/components/c_negation.py @@ -0,0 +1,125 @@ +"""Component C — Negation + temporality classifier. + +Lightweight rule-based detector. Looks at a window of tokens preceding each span +for negation cues ("no", "not", "denies", "without", "ruled out", "r/o", "r/o'd"), +family-history cues ("father", "mother", "brother", "sister", "family history of"), +and temporality cues ("history of", "h/o", "prior", "previous", "past", "resolved"). + +This is a conservative placeholder. Swap for `medspacy` or a clinical-BERT +classifier when the ML extras are installed. + +Critical correctness rule: **negated or family-history-only mentions must be +filtered out before HCC attribution.** This module provides the labels; the +orchestrator does the filter. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from app.pipeline.components.a_segmenter import section_for_offset +from app.pipeline.types import Note, Span, SpanType, Temporality + +_NEGATION_CUES = [ + "no ", "not ", "denies ", "denied ", "without ", + "ruled out", "r/o ", "negative for", "no evidence of", "absence of", +] + +_FAMILY_CUES = [ + "father", "mother", "brother", "sister", "son", "daughter", + "family history of", "fhx of", "family hx of", + "paternal", "maternal", +] + +_HISTORICAL_CUES = [ + "history of", "h/o ", "hx of", "prior", "previous", "previously", "past", + "resolved", "s/p ", "status post", "former", +] + +_HYPOTHETICAL_CUES = [ + "if ", "should ", "would ", "possibly", "suspected", "possible", + "r/o", "consider", "workup for", +] + +WINDOW_CHARS = 40 # characters of prefix context to scan + + +@dataclass +class NegationTemporalityClassifier: + window_chars: int = WINDOW_CHARS + + def __call__(self, note: Note, spans: list[Span]) -> list[Span]: + classified: list[Span] = [] + for span in spans: + # Only apply to Conditions / Findings; medications/labs/procedures rarely need this. + if span.span_type not in (SpanType.CONDITION, SpanType.FINDING): + classified.append(span) + continue + + context_start = max(0, span.start - self.window_chars) + context = note.text[context_start : span.start].lower() + + # Family history: also trust the section kind. + section_kind = span.section_kind or section_for_offset(note, span.start) + temporality = Temporality.CURRENT + negated = False + + if section_kind == "family_history" or any(c in context for c in _FAMILY_CUES): + temporality = Temporality.FAMILY_HISTORY + elif any(c in context for c in _NEGATION_CUES): + negated = True + elif any(c in context for c in _HYPOTHETICAL_CUES): + temporality = Temporality.HYPOTHETICAL + elif ( + section_kind == "past_medical_history" + or any(c in context for c in _HISTORICAL_CUES) + ): + temporality = Temporality.HISTORICAL + + classified.append( + span.model_copy( + update={ + "negated": negated, + "temporality": temporality, + "section_kind": section_kind, + } + ) + ) + return classified + + +# Heuristic: which (negation, temporality) combinations may contribute to an active HCC? +_HISTORICAL_CHRONIC_WHITELIST = { + # Terms that stay codable even when marked historical — chronic conditions + # typically remain active HCCs unless the note explicitly says "resolved". + "diabetes", "dm", "chf", "heart failure", "congestive heart failure", + "copd", "ckd", "chronic kidney disease", "atrial fibrillation", "afib", + "cirrhosis", "hepatitis", "dialysis", +} + + +def _is_chronic_historical_keeper(span: Span) -> bool: + low = span.text.lower() + return any(tok in low for tok in _HISTORICAL_CHRONIC_WHITELIST) + + +def is_attributable(span: Span) -> bool: + """Should this span be considered for HCC attribution? + + Rules (conservative): + - Negated: never. + - Family history: never. + - Hypothetical (r/o, workup): never. + - Historical: only if the underlying condition is chronic (whitelist). + - Current: always. + """ + if span.negated: + return False + if span.temporality == Temporality.FAMILY_HISTORY: + return False + if span.temporality == Temporality.HYPOTHETICAL: + return False + if span.temporality == Temporality.HISTORICAL: + return _is_chronic_historical_keeper(span) + return True diff --git a/app/pipeline/components/d_linker.py b/app/pipeline/components/d_linker.py new file mode 100644 index 0000000000000000000000000000000000000000..302dd933fd329a78e7cda3622f72571e2e403e51 --- /dev/null +++ b/app/pipeline/components/d_linker.py @@ -0,0 +1,144 @@ +"""Component D — ICD-10 linker. + +Two implementations: + * `ExactMatchLinker`: string-normalization + substring lookup against the staged + ICD-10 description CSV. Cheap; runs in CI. + * `BiEncoderLinker`: placeholder — will wrap SapBERT / MedCPT embeddings for real + retrieval-reranking. Not implemented yet. + +The linker emits top-k ICD-10 candidates per Condition span with a calibratable +confidence score. For v1 we emit just the best match. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path + +from app.pipeline.types import ICD10Link, Span, SpanType + + +def _normalize(text: str) -> str: + text = text.lower() + text = re.sub(r"[^a-z0-9 ]", " ", text) + text = re.sub(r"\s+", " ", text).strip() + return text + + +@dataclass +class ExactMatchLinker: + reference_dir: Path = field(default_factory=lambda: Path("data/reference")) + _desc_by_code: dict[str, str] = field(default_factory=dict, init=False) + _code_by_norm: dict[str, str] = field(default_factory=dict, init=False) + _aliases: dict[str, str] = field(default_factory=dict, init=False) + + def __post_init__(self) -> None: + desc_file = self.reference_dir / "icd10_cm_descriptions.csv" + if desc_file.exists(): + with desc_file.open() as f: + next(f, None) + for line in f: + parts = line.rstrip("\n").split(",", 1) + if len(parts) != 2: + continue + code = parts[0].strip() + desc = parts[1].strip().strip('"') + self._desc_by_code[code] = desc + self._code_by_norm[_normalize(desc)] = code + + # Hand-curated aliases so surface forms like "DM2" map to E11.9. + self._aliases = { + "diabetes": "E11.9", + "dm": "E11.9", + "dm2": "E11.9", + "t2dm": "E11.9", + "t1dm": "E10.9", + "type 2 diabetes": "E11.9", + "type 2 diabetes mellitus": "E11.9", + "type 1 diabetes": "E10.9", + "diabetic neuropathy": "E11.42", + "diabetic nephropathy": "E11.21", + "diabetes with neuropathy": "E11.40", + "diabetes with hyperglycemia": "E11.65", + "heart failure": "I50.9", + "chf": "I50.9", + "congestive heart failure": "I50.9", + "chronic systolic heart failure": "I50.22", + "chronic diastolic heart failure": "I50.32", + "copd": "J44.9", + "chronic obstructive pulmonary disease": "J44.9", + "copd exacerbation": "J44.1", + "ckd": "N18.30", + "ckd stage 3": "N18.30", + "ckd stage 3a": "N18.31", + "ckd stage 3b": "N18.32", + "ckd stage 4": "N18.4", + "ckd stage 5": "N18.5", + "esrd": "N18.6", + "end stage renal disease": "N18.6", + "dialysis": "Z99.2", + "atrial fibrillation": "I48.91", + "afib": "I48.91", + "a-fib": "I48.91", + "paroxysmal atrial fibrillation": "I48.0", + "chronic atrial fibrillation": "I48.2", + "unstable angina": "I20.0", + "angina": "I20.9", + "cirrhosis": "K74.60", + "alcoholic cirrhosis": "K70.30", + "hep c": "B18.2", + "hepatitis c": "B18.2", + "chronic hepatitis c": "B18.2", + "peripheral vascular disease": "I73.9", + "pvd": "I73.9", + "pulmonary fibrosis": "J84.10", + "idiopathic pulmonary fibrosis": "J84.112", + } + + def __call__(self, spans: list[Span]) -> list[ICD10Link]: + out: list[ICD10Link] = [] + for span in spans: + if span.span_type != SpanType.CONDITION: + continue + + norm = _normalize(span.text) + code: str | None = None + conf: float = 0.0 + + # 1. Alias hit (high confidence) + if norm in self._aliases: + code = self._aliases[norm] + conf = 0.85 + # 2. Exact description match + elif norm in self._code_by_norm: + code = self._code_by_norm[norm] + conf = 0.90 + else: + # 3. Best substring overlap against description index + best_code, best_score = None, 0.0 + for desc_norm, c in self._code_by_norm.items(): + if norm and (norm in desc_norm or desc_norm in norm): + score = min(len(norm), len(desc_norm)) / max(len(norm), len(desc_norm), 1) + if score > best_score: + best_code, best_score = c, score + if best_code is not None and best_score >= 0.4: + code = best_code + conf = round(0.5 + 0.4 * best_score, 3) + + if code is None: + continue + + out.append( + ICD10Link( + icd10=code, + description=self._desc_by_code.get(code, ""), + linker_confidence=conf, + condition_span=span, + ) + ) + return out + + +def default_linker() -> ExactMatchLinker: + return ExactMatchLinker() diff --git a/app/pipeline/components/e_crosswalk.py b/app/pipeline/components/e_crosswalk.py new file mode 100644 index 0000000000000000000000000000000000000000..6a9a2ab4cbedf993e92095bef1637d174ef32269 --- /dev/null +++ b/app/pipeline/components/e_crosswalk.py @@ -0,0 +1,192 @@ +"""Component E — ICD-10 → HCC v28 crosswalk + hierarchy trumping. + +Pure deterministic lookup over the CMS-HCC v28 tables staged in data/reference/. +No ML. Hierarchy trumping follows the CMS rule: within a disease group, the +highest-ranked (lowest `hierarchy_rank` number) HCC present in the candidate set +trumps all lower-ranked ones in the same group. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import duckdb + +from app.pipeline.types import ( + HCCCandidate, + HierarchyState, + ICD10Link, +) + + +@dataclass(frozen=True) +class HCCReference: + hcc_id: str + label: str + hierarchy_group: str + hierarchy_rank: int + raf_community_non_dual_aged: float + raf_community_dual_aged: float + raf_institutional: float + + +class HCCCrosswalk: + """Thin wrapper around the three CSVs under data/reference/. + + The crosswalk is small enough to keep entirely in memory. DuckDB is used only + for the initial CSV load + join so we get schema validation for free. + """ + + def __init__(self, reference_dir: Path | str = "data/reference") -> None: + self.reference_dir = Path(reference_dir) + con = duckdb.connect(":memory:") + con.execute( + f"CREATE TABLE labels AS SELECT * FROM read_csv_auto('{self.reference_dir}/hcc_v28_labels.csv')" + ) + con.execute( + f"CREATE TABLE crosswalk AS SELECT * FROM read_csv_auto('{self.reference_dir}/hcc_v28_crosswalk.csv')" + ) + con.execute( + f"CREATE TABLE hierarchy AS SELECT * FROM read_csv_auto('{self.reference_dir}/hcc_v28_hierarchy.csv')" + ) + con.execute( + f"CREATE TABLE icd10 AS SELECT * FROM read_csv_auto('{self.reference_dir}/icd10_cm_descriptions.csv')" + ) + + self._labels: dict[str, HCCReference] = {} + for row in con.execute("SELECT * FROM labels").fetchall(): + ref = HCCReference( + hcc_id=row[0], + label=row[1], + hierarchy_group=row[2], + hierarchy_rank=int(row[3]), + raf_community_non_dual_aged=float(row[4]), + raf_community_dual_aged=float(row[5]), + raf_institutional=float(row[6]), + ) + self._labels[ref.hcc_id] = ref + + self._icd_to_hcc: dict[str, str] = {} + for row in con.execute("SELECT icd10_code, hcc_id FROM crosswalk").fetchall(): + self._icd_to_hcc[row[0]] = row[1] + + self._icd_desc: dict[str, str] = dict( + con.execute("SELECT icd10_code, description FROM icd10").fetchall() + ) + + # hierarchy_group -> { hcc_id: rank } + self._group_members: dict[str, dict[str, int]] = {} + for ref in self._labels.values(): + self._group_members.setdefault(ref.hierarchy_group, {})[ref.hcc_id] = ref.hierarchy_rank + + # Explicit trumping edges (higher -> lower). We have implicit trumping via rank + # already, but the hierarchy.csv is the source of truth per CMS — so we load + # both and intersect during resolution. + self._explicit_trumps: dict[str, set[str]] = {} + for row in con.execute( + "SELECT higher_hcc, trumps_lower_hcc FROM hierarchy" + ).fetchall(): + self._explicit_trumps.setdefault(row[0], set()).add(row[1]) + + con.close() + + # ---------------------------------------------------------------- lookup + + def icd10_description(self, code: str) -> str: + return self._icd_desc.get(code, "") + + def hcc_for_icd10(self, icd10: str) -> str | None: + return self._icd_to_hcc.get(icd10) + + def hcc_reference(self, hcc_id: str) -> HCCReference | None: + return self._labels.get(hcc_id) + + def all_hccs(self) -> list[HCCReference]: + return list(self._labels.values()) + + def all_icd10_in_crosswalk(self) -> list[str]: + return list(self._icd_to_hcc.keys()) + + # ----------------------------------------------------------- candidate build + + def build_candidates( + self, + links: list[ICD10Link], + raf_segment: str = "community_non_dual_aged", + ) -> list[HCCCandidate]: + """Convert ICD-10 links to HCC candidates, grouping links that map to the same HCC. + + `raf_segment` ∈ {community_non_dual_aged, community_dual_aged, institutional}. + """ + grouped: dict[str, list[ICD10Link]] = {} + for link in links: + hcc_id = self.hcc_for_icd10(link.icd10) + if hcc_id is None: + continue + grouped.setdefault(hcc_id, []).append(link) + + candidates: list[HCCCandidate] = [] + for hcc_id, hcc_links in grouped.items(): + ref = self._labels[hcc_id] + raf_weight = { + "community_non_dual_aged": ref.raf_community_non_dual_aged, + "community_dual_aged": ref.raf_community_dual_aged, + "institutional": ref.raf_institutional, + }[raf_segment] + + # Raw confidence: max linker confidence across supporting links, + # tempered by the NER confidence of the best span. + raw = max( + link.linker_confidence * link.condition_span.ner_confidence + for link in hcc_links + ) + + candidates.append( + HCCCandidate( + hcc=hcc_id, + hcc_label=ref.label, + raf_weight=raf_weight, + hierarchy=HierarchyState( + group=ref.hierarchy_group, + rank=ref.hierarchy_rank, + ), + icd10_links=hcc_links, + confidence_raw=raw, + confidence_calibrated=raw, # overwritten by component G + ) + ) + + return candidates + + # ------------------------------------------------------------ trumping + + def apply_trumping(self, candidates: list[HCCCandidate]) -> list[HCCCandidate]: + """Mark lower-ranked candidates in the same hierarchy group as trumped. + + Uses the explicit edges from `hcc_v28_hierarchy.csv` as the source of truth + AND the rank ordering as a fallback (a v28 hierarchy file may omit rows for + transitive closures). + """ + present = {c.hcc for c in candidates} + + for cand in candidates: + trumped_by: str | None = None + for other in candidates: + if other.hcc == cand.hcc: + continue + if other.hierarchy.group != cand.hierarchy.group: + continue + # Explicit file edge + if cand.hcc in self._explicit_trumps.get(other.hcc, set()): + trumped_by = other.hcc + break + # Rank-based: stricter rank (smaller number) trumps within group. + if other.hierarchy.rank < cand.hierarchy.rank: + trumped_by = other.hcc + break + if trumped_by is not None and trumped_by in present: + cand.hierarchy.trumped = True + cand.hierarchy.trumped_by = trumped_by + + return candidates diff --git a/app/pipeline/components/f_meat.py b/app/pipeline/components/f_meat.py new file mode 100644 index 0000000000000000000000000000000000000000..cf7a878904ec38d1d18b141f956cfdaad7bba202 --- /dev/null +++ b/app/pipeline/components/f_meat.py @@ -0,0 +1,190 @@ +"""Component F — MEAT extractor. + +Two implementations: + * `RuleBasedMEATExtractor`: scans the note for keyword / section cues per MEAT + category and emits exact-substring quotes. Low ceiling but hallucination-proof. + This is the CI-friendly baseline and the fallback when ML deps aren't loaded. + * `LLMMEATExtractor`: (stub) — will prompt a small instruction-tuned LLM + (Phi-4-mini or Qwen2.5-3B-Instruct) with a JSON schema constrained decoder. + Post-decode substring check is mandatory (enforced by Component I). + +The extractor never guarantees entailment — that's Component I's job. It just +produces candidate quotes tagged M/E/A/T. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Protocol + +from app.pipeline.types import ( + HCCCandidate, + MEATCategory, + MEATEvidence, + Note, + NoteSection, +) + + +class MEATExtractor(Protocol): + def __call__( + self, note: Note, candidate: HCCCandidate + ) -> list[MEATEvidence]: ... + + +# --------------------------------------------------------------------------- +# Keyword cues. Tuned to be high-precision / low-recall — missing a category is +# preferable to a spurious hit, since the UI surfaces "insufficient MEAT" loud. +# --------------------------------------------------------------------------- + +_MONITOR_CUES = [ + "monitor", "monitoring", "tracked", "tracking", "follow up", "f/u", + "checked", "trending", "stable on", "worsening", "improving", +] + +_EVALUATE_CUES = [ + "a1c", "hba1c", "lab", "labs", "creatinine", "egfr", "bnp", + "echocardiogram", "echo", "ekg", "ecg", "fundoscopy", "fundus exam", + "x-ray", "chest x-ray", "ct scan", "mri", "ultrasound", "reviewed", + "result", "results", +] + +_ASSESS_CUES = [ + "impression", "assessment", "diagnosis", "diagnosed with", + "consistent with", "active", "uncontrolled", "controlled", + "stable", "stage", +] + +_TREAT_CUES = [ + "prescribed", "started on", "continue", "continued", "titrated", + "increased", "decreased", "discontinued", "referred to", "referral", + "insulin", "metformin", "diuretic", "inhaler", "dialysis", +] + +# Sentence splitter: crude but consistent. +_SENT_RE = re.compile(r"(?<=[.!?])\s+") + + +def _sentences_with_offsets(text: str) -> list[tuple[int, int, str]]: + out: list[tuple[int, int, str]] = [] + cursor = 0 + for piece in _SENT_RE.split(text): + if not piece.strip(): + cursor += len(piece) + 1 + continue + start = text.find(piece, cursor) + if start == -1: + continue + end = start + len(piece) + out.append((start, end, piece)) + cursor = end + return out + + +def _cue_hits(sentence_lower: str, cues: list[str]) -> bool: + return any(cue in sentence_lower for cue in cues) + + +def _mentions_candidate(sentence_lower: str, candidate: HCCCandidate) -> bool: + """Loose textual match: does the sentence plausibly refer to the candidate condition?""" + label = candidate.hcc_label.lower() + # Strip hedging words that don't help match. + label_tokens = [ + tok for tok in re.split(r"[^a-z0-9]+", label) if tok and len(tok) > 3 + ] + if any(tok in sentence_lower for tok in label_tokens): + return True + # Also accept a hit on one of the supporting ICD-10 description tokens. + for link in candidate.icd10_links: + desc = link.description.lower() + for tok in re.split(r"[^a-z0-9]+", desc): + if len(tok) > 4 and tok in sentence_lower: + return True + surface = link.condition_span.text.lower() + if len(surface) >= 3 and surface in sentence_lower: + return True + return False + + +@dataclass +class RuleBasedMEATExtractor: + extractor_confidence: float = 0.40 # reflects that this is a weak baseline + + def __call__(self, note: Note, candidate: HCCCandidate) -> list[MEATEvidence]: + sentences = _sentences_with_offsets(note.text) + out: list[MEATEvidence] = [] + + for start, end, sent in sentences: + sent_lower = sent.lower() + if not _mentions_candidate(sent_lower, candidate): + continue + + section_kind = _section_kind_for(note.sections, start) + + for category, cues in ( + (MEATCategory.MONITOR, _MONITOR_CUES), + (MEATCategory.EVALUATE, _EVALUATE_CUES), + (MEATCategory.ASSESS, _ASSESS_CUES), + (MEATCategory.TREAT, _TREAT_CUES), + ): + if _cue_hits(sent_lower, cues): + out.append( + MEATEvidence( + category=category, + quote=sent, + start=start, + end=end, + section_kind=section_kind, + extractor_confidence=self.extractor_confidence, + verified=False, # Component I decides this + ) + ) + + # If we found NO assessment evidence but the candidate's condition appears in + # an assessment/plan section, surface that sentence as weak Assess evidence. + if not any(ev.category == MEATCategory.ASSESS for ev in out): + for section in note.sections: + if section.kind not in ("assessment", "assessment_and_plan", "plan"): + continue + for start, end, sent in sentences: + if start < section.start or end > section.end: + continue + if _mentions_candidate(sent.lower(), candidate): + out.append( + MEATEvidence( + category=MEATCategory.ASSESS, + quote=sent, + start=start, + end=end, + section_kind=section.kind, + extractor_confidence=0.30, + verified=False, + ) + ) + break + + return _dedupe_meat(out) + + +def _section_kind_for(sections: list[NoteSection], offset: int) -> str | None: + for s in sections: + if s.start <= offset < s.end: + return s.kind + return None + + +def _dedupe_meat(evidences: list[MEATEvidence]) -> list[MEATEvidence]: + seen: set[tuple[str, int, int]] = set() + out: list[MEATEvidence] = [] + for ev in evidences: + key = (ev.category.value, ev.start, ev.end) + if key in seen: + continue + seen.add(key) + out.append(ev) + return out + + +def default_meat_extractor() -> RuleBasedMEATExtractor: + return RuleBasedMEATExtractor() diff --git a/app/pipeline/components/g_calibrator.py b/app/pipeline/components/g_calibrator.py new file mode 100644 index 0000000000000000000000000000000000000000..4259cd33b1044248461694823770888b2b81afe3 --- /dev/null +++ b/app/pipeline/components/g_calibrator.py @@ -0,0 +1,70 @@ +"""Component G — Calibrator. + +Maps raw per-HCC confidence scores to a calibrated probability. Two paths: + + * `IdentityCalibrator`: pass-through. Used until enough held-out data exists. + * `IsotonicCalibrator`: per-HCC (or per-frequency-bucket if sparse) isotonic + regression fit on held-out Synthea. Persist to `data/models/calibrator.pkl` + and load at service startup. + +We do NOT store a stale calibrator in git. The calibrator training script +(`scripts/train_calibrator.py` — TBD) writes the file; the Docker image can +copy it from a mounted volume. +""" + +from __future__ import annotations + +import json +import pickle +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol + + +class Calibrator(Protocol): + def calibrate(self, hcc_id: str, raw: float) -> float: ... + + +@dataclass +class IdentityCalibrator: + def calibrate(self, hcc_id: str, raw: float) -> float: # noqa: ARG002 + return raw + + +@dataclass +class IsotonicCalibrator: + """Per-HCC isotonic regressors persisted to disk. + + The stored pickle is a dict[str, sklearn.isotonic.IsotonicRegression]. + Unknown HCCs fall through to the `fallback_bucket` (rare / medium / frequent). + """ + + model_path: Path + fallback_bucket: str = "medium" + _per_hcc: dict = field(default_factory=dict, init=False) + _buckets: dict = field(default_factory=dict, init=False) + _meta: dict = field(default_factory=dict, init=False) + + def __post_init__(self) -> None: + if self.model_path.exists(): + with self.model_path.open("rb") as f: + payload = pickle.load(f) + self._per_hcc = payload.get("per_hcc", {}) + self._buckets = payload.get("buckets", {}) + self._meta = payload.get("meta", {}) + + def calibrate(self, hcc_id: str, raw: float) -> float: + reg = self._per_hcc.get(hcc_id) + if reg is None: + reg = self._buckets.get(self.fallback_bucket) + if reg is None: + return raw + return float(reg.predict([raw])[0]) + + def to_metadata_json(self) -> str: + return json.dumps({"meta": self._meta, "hccs_known": list(self._per_hcc)}) + + +def default_calibrator() -> Calibrator: + # Until a trained model is produced, pass through raw scores. + return IdentityCalibrator() diff --git a/app/pipeline/components/h_risk.py b/app/pipeline/components/h_risk.py new file mode 100644 index 0000000000000000000000000000000000000000..650055c2968616d391906bfa14fcd5101bdcdafb --- /dev/null +++ b/app/pipeline/components/h_risk.py @@ -0,0 +1,103 @@ +"""Component H — over-coding (RADV) risk heuristic. + +Transparent, additive weighting — not a black box. Every reason shown in the UI +is traceable to one of the score components below. Weights are deliberately +conservative; calibration is the job of Component G, not this heuristic. +""" + +from __future__ import annotations + +from app.pipeline.types import HCCCandidate, RiskAssessment + +# Weights sum to 1.0. Keep these named so the UI can surface them. +WEIGHT_MEAT_COVERAGE = 0.40 +WEIGHT_NER_CONFIDENCE = 0.20 +WEIGHT_LINKER_CONFIDENCE = 0.20 +WEIGHT_RAF_PRESSURE = 0.10 +WEIGHT_HIERARCHY_SUPPORT = 0.10 + +# A high RAF weight is *itself* audit-risk because RADV samples high-value codes +# more aggressively. We normalize against a reference high-value weight. +RAF_HIGH_REFERENCE = 0.40 + +LOW_THRESHOLD = 0.70 +MEDIUM_THRESHOLD = 0.45 + + +def _raf_pressure(raf_weight: float) -> float: + """Return a 0..1 score: higher RAF = more pressure = worse (needs to be weighed against).""" + # Normalize: 0 RAF => 1.0 (no pressure), RAF_HIGH_REFERENCE => 0.0 (max pressure). + return max(0.0, 1.0 - min(raf_weight / RAF_HIGH_REFERENCE, 1.0)) + + +def _hierarchy_support(cand: HCCCandidate) -> tuple[float, str | None]: + """Return support score + reason. 1.0 = this HCC is not trumped; 0.0 = trumped.""" + if cand.hierarchy.trumped: + return 0.0, f"Trumped by {cand.hierarchy.trumped_by} in {cand.hierarchy.group} hierarchy" + return 1.0, None + + +def assess_risk(cand: HCCCandidate) -> RiskAssessment: + meat_coverage = cand.meat_coverage_ratio() + + # Use the best NER confidence from any condition span supporting any ICD10 link. + ner_conf = ( + max((lnk.condition_span.ner_confidence for lnk in cand.icd10_links), default=0.0) + if cand.icd10_links + else 0.0 + ) + linker_conf = ( + max((lnk.linker_confidence for lnk in cand.icd10_links), default=0.0) + if cand.icd10_links + else 0.0 + ) + raf_pressure = _raf_pressure(cand.raf_weight) + hier_support, hier_reason = _hierarchy_support(cand) + + score = ( + WEIGHT_MEAT_COVERAGE * meat_coverage + + WEIGHT_NER_CONFIDENCE * ner_conf + + WEIGHT_LINKER_CONFIDENCE * linker_conf + + WEIGHT_RAF_PRESSURE * raf_pressure + + WEIGHT_HIERARCHY_SUPPORT * hier_support + ) + + reasons: list[str] = [] + coverage_dict = cand.meat_coverage() + missing = [c.name.title() for c, filled in coverage_dict.items() if not filled] + if missing: + reasons.append(f"Missing MEAT categories: {', '.join(missing)}") + else: + reasons.append("All four MEAT categories verified in note") + + if ner_conf < 0.5: + reasons.append(f"Weak NER confidence on supporting span ({ner_conf:.2f})") + if linker_conf < 0.5: + reasons.append(f"Weak ICD-10 linker confidence ({linker_conf:.2f})") + if cand.raf_weight >= RAF_HIGH_REFERENCE: + reasons.append( + f"High RAF weight ({cand.raf_weight:.3f}) — RADV audits sample high-value codes more" + ) + if hier_reason is not None: + reasons.append(hier_reason) + + level: str + if score >= LOW_THRESHOLD: + level = "low" + elif score >= MEDIUM_THRESHOLD: + level = "medium" + else: + level = "high" + + return RiskAssessment( + level=level, + score=round(score, 4), + reasons=reasons, + components={ + "meat_coverage": round(meat_coverage, 4), + "ner_confidence": round(ner_conf, 4), + "linker_confidence": round(linker_conf, 4), + "raf_pressure": round(raf_pressure, 4), + "hierarchy_support": round(hier_support, 4), + }, + ) diff --git a/app/pipeline/components/i_verifier.py b/app/pipeline/components/i_verifier.py new file mode 100644 index 0000000000000000000000000000000000000000..813b8fe5abb96887866f43ab5a23cd88739d3da7 --- /dev/null +++ b/app/pipeline/components/i_verifier.py @@ -0,0 +1,117 @@ +"""Component I — faithfulness verifier. + +Enforces two invariants for every MEAT evidence span: + 1. The quoted text MUST be an exact substring of the source note. + 2. Optional NLI check: the quote entails the MEAT category it is claimed for. + +If (1) fails, the evidence is dropped and that MEAT category is left empty. +If (2) fails below a threshold, the evidence is kept but `verified=False` and +the UI will show it as "weak evidence". + +The NLI model is injected at construction time so the verifier can run without +any ML deps loaded (returns None for `nli_entailment` in that case). +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Protocol + +from app.pipeline.types import HCCCandidate, MEATCategory, MEATEvidence, Note + + +class NLIVerifier(Protocol): + def __call__( + self, premise: str, hypothesis: str + ) -> float: # returns entailment probability + ... + + +MEAT_HYPOTHESIS_TEMPLATES: dict[MEATCategory, str] = { + MEATCategory.MONITOR: ( + "The clinician is monitoring signs or symptoms of {condition} " + "(e.g., tracking labs, vitals, or disease progression)." + ), + MEATCategory.EVALUATE: ( + "The clinician reviewed a test, exam, or study result relevant to {condition}." + ), + MEATCategory.ASSESS: ( + "The clinician documented a clinical impression, status, or assessment of {condition}." + ), + MEATCategory.TREAT: ( + "The clinician prescribed, continued, or adjusted a treatment (medication, procedure, " + "referral, or therapy) for {condition}." + ), +} + +# Below this entailment probability, the evidence is considered unverified. +NLI_ENTAILMENT_THRESHOLD = 0.5 + + +class FaithfulnessVerifier: + """Verify that every MEAT quote (a) is an exact substring and (b) entails the MEAT category.""" + + def __init__( + self, + nli: NLIVerifier | None = None, + entailment_threshold: float = NLI_ENTAILMENT_THRESHOLD, + ) -> None: + self._nli = nli + self._threshold = entailment_threshold + + def verify_substring(self, note: Note, ev: MEATEvidence) -> bool: + """Cheap + non-negotiable: the quote must be a byte-exact substring of the note.""" + # Prefer offset-based check; fall back to substring search. + if 0 <= ev.start < ev.end <= len(note.text): + if note.text[ev.start : ev.end] == ev.quote: + return True + return ev.quote in note.text + + def verify_evidence( + self, note: Note, condition_text: str, ev: MEATEvidence + ) -> MEATEvidence: + if not self.verify_substring(note, ev): + # Drop it: caller must filter these out or mark MEAT category insufficient. + return ev.model_copy(update={"verified": False, "nli_entailment": None}) + + if self._nli is None: + return ev.model_copy(update={"verified": True, "nli_entailment": None}) + + hypothesis = MEAT_HYPOTHESIS_TEMPLATES[ev.category].format(condition=condition_text) + entailment = float(self._nli(ev.quote, hypothesis)) + return ev.model_copy( + update={ + "verified": entailment >= self._threshold, + "nli_entailment": entailment, + } + ) + + def verify_candidate( + self, note: Note, candidate: HCCCandidate + ) -> tuple[HCCCandidate, int]: + """Verify every MEAT evidence item on the candidate. + + Returns the candidate with a filtered/annotated evidence list and the number + of substring-failing evidence items that were dropped (for telemetry). + """ + condition_text = candidate.hcc_label.lower() + drops = 0 + verified: list[MEATEvidence] = [] + for ev in candidate.meat_evidence: + if not self.verify_substring(note, ev): + drops += 1 + continue + verified.append(self.verify_evidence(note, condition_text, ev)) + return ( + candidate.model_copy(update={"meat_evidence": verified}), + drops, + ) + + +def identity_nli(premise: str, hypothesis: str) -> float: # noqa: ARG001 + """A debug NLI that returns 1.0 always. Useful in tests that don't care about entailment.""" + return 1.0 + + +def make_default_verifier(nli: Callable[[str, str], float] | None = None) -> FaithfulnessVerifier: + return FaithfulnessVerifier(nli=nli) diff --git a/app/pipeline/orchestrator.py b/app/pipeline/orchestrator.py new file mode 100644 index 0000000000000000000000000000000000000000..e61072cb3f567c63e048a24b4d8d75a63312a90d --- /dev/null +++ b/app/pipeline/orchestrator.py @@ -0,0 +1,124 @@ +"""Pipeline orchestrator — wires components A..I into a single Note → CodingResult step. + +Component ownership: + A Section Segmenter -> app.pipeline.components.a_segmenter + B Clinical NER -> app.pipeline.components.b_ner + C Negation + Temporality -> app.pipeline.components.c_negation + D ICD-10 Linker -> app.pipeline.components.d_linker + E HCC Crosswalk + Trumping -> app.pipeline.components.e_crosswalk + F MEAT Extractor -> app.pipeline.components.f_meat + G Calibrator -> app.pipeline.components.g_calibrator + H Over-coding Risk -> app.pipeline.components.h_risk + I Faithfulness Verifier -> app.pipeline.components.i_verifier + +The orchestrator is deliberately stateless aside from the reference-data and +model handles it caches at construction time. One instance should be held at +the FastAPI process level and reused across requests. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from pathlib import Path + +from app.pipeline.components import a_segmenter, b_ner, c_negation, d_linker, f_meat, g_calibrator +from app.pipeline.components.b_ner import NER, default_ner +from app.pipeline.components.c_negation import NegationTemporalityClassifier, is_attributable +from app.pipeline.components.d_linker import ExactMatchLinker, default_linker +from app.pipeline.components.e_crosswalk import HCCCrosswalk +from app.pipeline.components.f_meat import MEATExtractor, default_meat_extractor +from app.pipeline.components.g_calibrator import Calibrator, default_calibrator +from app.pipeline.components.h_risk import assess_risk +from app.pipeline.components.i_verifier import FaithfulnessVerifier, make_default_verifier +from app.pipeline.types import CodingResult, HCCCandidate, Note + + +@dataclass +class PipelineConfig: + reference_dir: Path = field(default_factory=lambda: Path("data/reference")) + raf_segment: str = "community_non_dual_aged" + + +@dataclass +class Pipeline: + config: PipelineConfig = field(default_factory=PipelineConfig) + ner: NER | None = None + linker: ExactMatchLinker | None = None + crosswalk: HCCCrosswalk | None = None + meat_extractor: MEATExtractor | None = None + calibrator: Calibrator | None = None + negation: NegationTemporalityClassifier | None = None + verifier: FaithfulnessVerifier | None = None + + def __post_init__(self) -> None: + self.ner = self.ner or default_ner() + self.linker = self.linker or default_linker() + self.crosswalk = self.crosswalk or HCCCrosswalk(self.config.reference_dir) + self.meat_extractor = self.meat_extractor or default_meat_extractor() + self.calibrator = self.calibrator or default_calibrator() + self.negation = self.negation or NegationTemporalityClassifier() + self.verifier = self.verifier or make_default_verifier() + + def run(self, note: Note) -> CodingResult: + t0 = time.perf_counter() + + # A — segment + note = a_segmenter.segment(note) + + # B — NER + spans = self.ner(note) + + # C — negation + temporality; then drop non-attributable spans + spans = self.negation(note, spans) + eligible = [s for s in spans if is_attributable(s)] + + # D — ICD-10 linking (conditions only) + links = self.linker(eligible) + + # E — crosswalk + trumping + candidates = self.crosswalk.build_candidates(links, raf_segment=self.config.raf_segment) + candidates = self.crosswalk.apply_trumping(candidates) + + # F — MEAT extraction, then I — faithfulness verification + verifier_drops = 0 + enriched: list[HCCCandidate] = [] + for cand in candidates: + evidence = self.meat_extractor(note, cand) + with_evidence = cand.model_copy(update={"meat_evidence": evidence}) + verified_cand, drops = self.verifier.verify_candidate(note, with_evidence) + verifier_drops += drops + enriched.append(verified_cand) + + # G — calibration + calibrated: list[HCCCandidate] = [] + for cand in enriched: + cal = self.calibrator.calibrate(cand.hcc, cand.confidence_raw) + calibrated.append(cand.model_copy(update={"confidence_calibrated": float(cal)})) + + # H — over-coding risk + with_risk = [c.model_copy(update={"risk": assess_risk(c)}) for c in calibrated] + + # Rank: non-trumped first, then by calibrated confidence × RAF weight. + with_risk.sort( + key=lambda c: ( + 0 if not c.hierarchy.trumped else 1, + -(c.confidence_calibrated * max(c.raf_weight, 0.01)), + ) + ) + + return CodingResult( + note_id=note.note_id, + candidates=with_risk, + verifier_drops=verifier_drops, + latency_ms=(time.perf_counter() - t0) * 1000.0, + model_refs={ + "ner": type(self.ner).__name__, + "linker": type(self.linker).__name__, + "meat_extractor": type(self.meat_extractor).__name__, + "calibrator": type(self.calibrator).__name__, + }, + ) + + +__all__ = ["Pipeline", "PipelineConfig"] diff --git a/app/pipeline/types.py b/app/pipeline/types.py new file mode 100644 index 0000000000000000000000000000000000000000..de6b073a5adbf344ac50f8c2b3aabaf90f3b43ea --- /dev/null +++ b/app/pipeline/types.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from enum import Enum +from typing import Literal + +from pydantic import BaseModel, Field, model_validator + + +class SpanType(str, Enum): + CONDITION = "Condition" + MEDICATION = "Medication" + LAB_TEST = "LabTest" + PROCEDURE = "Procedure" + FINDING = "Finding" + ANATOMY = "Anatomy" + + +class Temporality(str, Enum): + CURRENT = "current" + HISTORICAL = "historical" + HYPOTHETICAL = "hypothetical" + FAMILY_HISTORY = "family_history" + + +class MEATCategory(str, Enum): + MONITOR = "M" + EVALUATE = "E" + ASSESS = "A" + TREAT = "T" + + +SectionKind = Literal[ + "chief_complaint", + "hpi", + "ros", + "physical_exam", + "labs", + "medications", + "assessment", + "plan", + "assessment_and_plan", + "social_history", + "family_history", + "past_medical_history", + "other", +] + + +class NoteSection(BaseModel): + kind: SectionKind + text: str + start: int = Field(..., description="Character offset in original note where section begins.") + end: int = Field(..., description="Character offset where section ends (exclusive).") + + +class Note(BaseModel): + note_id: str + text: str + metadata: dict = Field(default_factory=dict) + sections: list[NoteSection] = Field(default_factory=list) + + def slice(self, start: int, end: int) -> str: + return self.text[start:end] + + +class Span(BaseModel): + start: int + end: int + text: str + span_type: SpanType + ner_confidence: float = Field(ge=0.0, le=1.0, default=1.0) + negated: bool = False + temporality: Temporality = Temporality.CURRENT + section_kind: SectionKind | None = None + + @model_validator(mode="after") + def _check_offsets(self) -> Span: + if self.end <= self.start: + raise ValueError(f"Span end must exceed start, got [{self.start}, {self.end}]") + return self + + +class ICD10Link(BaseModel): + icd10: str + description: str + linker_confidence: float = Field(ge=0.0, le=1.0) + condition_span: Span + + +class HierarchyState(BaseModel): + group: str = Field(..., description="Hierarchy group identifier per CMS v28.") + rank: int = Field(..., description="Severity rank within the group — lower number = more severe.") + trumped: bool = False + trumped_by: str | None = None + + +class MEATEvidence(BaseModel): + category: MEATCategory + quote: str + start: int + end: int + section_kind: SectionKind | None = None + extractor_confidence: float = Field(ge=0.0, le=1.0, default=0.0) + nli_entailment: float | None = Field(default=None, ge=0.0, le=1.0) + verified: bool = False + + +class RiskAssessment(BaseModel): + level: Literal["low", "medium", "high"] + score: float = Field(ge=0.0, le=1.0) + reasons: list[str] = Field(default_factory=list) + components: dict[str, float] = Field(default_factory=dict) + + +class HCCCandidate(BaseModel): + hcc: str = Field(..., description="HCC v28 identifier (e.g., 'HCC-19').") + hcc_label: str + raf_weight: float + hierarchy: HierarchyState + icd10_links: list[ICD10Link] + confidence_raw: float = Field(ge=0.0, le=1.0) + confidence_calibrated: float = Field(ge=0.0, le=1.0) + meat_evidence: list[MEATEvidence] = Field(default_factory=list) + risk: RiskAssessment | None = None + + def meat_coverage(self) -> dict[MEATCategory, bool]: + filled = {c: False for c in MEATCategory} + for ev in self.meat_evidence: + if ev.verified: + filled[ev.category] = True + return filled + + def meat_coverage_ratio(self) -> float: + return sum(self.meat_coverage().values()) / 4.0 + + +class CodingResult(BaseModel): + note_id: str + candidates: list[HCCCandidate] + verifier_drops: int = 0 + latency_ms: float | None = None + pipeline_version: str = "0.1.0" + model_refs: dict[str, str] = Field(default_factory=dict) + + def accepted(self) -> list[HCCCandidate]: + return [c for c in self.candidates if not c.hierarchy.trumped] diff --git a/data/reference/PROVENANCE.md b/data/reference/PROVENANCE.md new file mode 100644 index 0000000000000000000000000000000000000000..ec0ade5993a823173a68e58983371f70912362c9 --- /dev/null +++ b/data/reference/PROVENANCE.md @@ -0,0 +1,43 @@ +# Reference data provenance + +All files in this directory are derived from public CMS / CDC / NLM sources. +None contain PHI. All are redistributable under the terms of their upstream license. + +## Files + +| File | Upstream | License | Notes | +|---|---|---|---| +| `icd10_cm_descriptions.csv` | CDC/CMS ICD-10-CM annual release | Public domain (US government work) | Tabular list — code + title only | +| `hcc_v28_labels.csv` | CMS Risk Adjustment — HCC v28 model | Public | HCC id, label, hierarchy group, RAF weight by segment | +| `hcc_v28_crosswalk.csv` | CMS Risk Adjustment — 2024 Mapping | Public | ICD-10 → HCC v28 mapping | +| `hcc_v28_hierarchy.csv` | CMS Risk Adjustment — v28 Hierarchy File | Public | Trumping rules (disease-group-level) | + +## Current state + +**⚠ The seed files in this directory are hand-curated PLACEHOLDERS for the ~15 most common HCCs, +intended to let the pipeline + tests run end-to-end without a network call. Every RAF weight +marked `PLACEHOLDER` in the CSV comments must be replaced with the official CMS value before +publishing metrics.** + +To hydrate the full v28 tables, run: + +``` +python scripts/fetch_cms_hcc_v28.py --out data/reference/ +``` + +This script downloads: +1. ICD-10-CM descriptions from CDC's NCHS public FTP. +2. CMS-HCC v28 model files from the CMS Risk Adjustment page + (https://www.cms.gov/medicare/payment/medicare-advantage-rates-statistics/risk-adjustment). + +## How to verify the seed data + +Run `pytest tests/test_reference_data.py` — it asserts: +- Every HCC in `hcc_v28_labels.csv` appears in `hcc_v28_crosswalk.csv` at least once. +- Every ICD-10 in `hcc_v28_crosswalk.csv` has a description in `icd10_cm_descriptions.csv`. +- Every HCC hierarchy group in `hcc_v28_hierarchy.csv` has members in `hcc_v28_labels.csv`. + +## License statement for downstream repos + +The crosswalk tables are works of the U.S. federal government and not copyrightable under +17 U.S.C. § 105. Redistribution with attribution to CMS is permitted. diff --git a/data/reference/hcc_v28_crosswalk.csv b/data/reference/hcc_v28_crosswalk.csv new file mode 100644 index 0000000000000000000000000000000000000000..267090af8577e3a1a6a0faa4b6ac5edd5987e442 --- /dev/null +++ b/data/reference/hcc_v28_crosswalk.csv @@ -0,0 +1,55 @@ +icd10_code,hcc_id,notes +E10.10,HCC-17,Type 1 diabetes mellitus with ketoacidosis without coma +E10.11,HCC-17,Type 1 diabetes mellitus with ketoacidosis with coma +E11.00,HCC-17,Type 2 diabetes mellitus with hyperosmolarity without NKHHC +E11.01,HCC-17,Type 2 diabetes mellitus with hyperosmolarity with coma +E10.21,HCC-18,Type 1 diabetes mellitus with diabetic nephropathy +E10.22,HCC-18,Type 1 diabetes mellitus with diabetic chronic kidney disease +E10.40,HCC-18,Type 1 diabetes mellitus with diabetic neuropathy unspecified +E10.42,HCC-18,Type 1 diabetes mellitus with diabetic polyneuropathy +E11.21,HCC-18,Type 2 diabetes mellitus with diabetic nephropathy +E11.22,HCC-18,Type 2 diabetes mellitus with diabetic chronic kidney disease +E11.40,HCC-18,Type 2 diabetes mellitus with diabetic neuropathy unspecified +E11.42,HCC-18,Type 2 diabetes mellitus with diabetic polyneuropathy +E11.65,HCC-18,Type 2 diabetes mellitus with hyperglycemia +E10.9,HCC-19,Type 1 diabetes mellitus without complications +E11.9,HCC-19,Type 2 diabetes mellitus without complications +K70.30,HCC-35,Alcoholic cirrhosis of liver without ascites +K70.31,HCC-35,Alcoholic cirrhosis of liver with ascites +K74.60,HCC-35,Unspecified cirrhosis of liver +K74.69,HCC-35,Other cirrhosis of liver +B18.2,HCC-37,Chronic viral hepatitis C +B18.0,HCC-37,Chronic viral hepatitis B with delta-agent +B18.1,HCC-37,Chronic viral hepatitis B without delta-agent +D68.32,HCC-48,Hemorrhagic disorder due to extrinsic circulating anticoagulants +D68.4,HCC-48,Acquired coagulation factor deficiency +I50.9,HCC-85,Heart failure unspecified +I50.20,HCC-85,Unspecified systolic (congestive) heart failure +I50.22,HCC-85,Chronic systolic (congestive) heart failure +I50.30,HCC-85,Unspecified diastolic (congestive) heart failure +I50.32,HCC-85,Chronic diastolic (congestive) heart failure +I50.40,HCC-85,Unspecified combined systolic and diastolic heart failure +I50.42,HCC-85,Chronic combined systolic and diastolic heart failure +I20.0,HCC-88,Unstable angina +I20.1,HCC-88,Angina pectoris with documented spasm +I20.8,HCC-88,Other forms of angina pectoris +I20.9,HCC-88,Angina pectoris unspecified +I48.0,HCC-96,Paroxysmal atrial fibrillation +I48.1,HCC-96,Persistent atrial fibrillation +I48.2,HCC-96,Chronic atrial fibrillation +I48.91,HCC-96,Unspecified atrial fibrillation +I70.209,HCC-108,Unspecified atherosclerosis of native arteries of extremities +I70.219,HCC-108,Atherosclerosis of native arteries of extremities with intermittent claudication +I73.9,HCC-108,Peripheral vascular disease unspecified +J44.0,HCC-111,COPD with acute lower respiratory infection +J44.1,HCC-111,COPD with acute exacerbation +J44.9,HCC-111,Chronic obstructive pulmonary disease unspecified +J84.10,HCC-112,Pulmonary fibrosis unspecified +J84.112,HCC-112,Idiopathic pulmonary fibrosis +Z99.2,HCC-134,Dependence on renal dialysis +N18.6,HCC-134,End stage renal disease +N18.5,HCC-136,Chronic kidney disease stage 5 +N18.4,HCC-137,Chronic kidney disease stage 4 (severe) +N18.30,HCC-138,Chronic kidney disease stage 3 unspecified +N18.31,HCC-138,Chronic kidney disease stage 3a +N18.32,HCC-138,Chronic kidney disease stage 3b diff --git a/data/reference/hcc_v28_hierarchy.csv b/data/reference/hcc_v28_hierarchy.csv new file mode 100644 index 0000000000000000000000000000000000000000..7ca652c61a4d9f04e3d19655a22026426447024c --- /dev/null +++ b/data/reference/hcc_v28_hierarchy.csv @@ -0,0 +1,12 @@ +hierarchy_group,higher_hcc,trumps_lower_hcc,notes +DIABETES,HCC-17,HCC-18,PLACEHOLDER — DM w/ acute complications trumps DM w/ chronic complications +DIABETES,HCC-17,HCC-19,PLACEHOLDER +DIABETES,HCC-18,HCC-19,PLACEHOLDER +LIVER,HCC-35,HCC-37,PLACEHOLDER +KIDNEY,HCC-134,HCC-136,PLACEHOLDER — dialysis status trumps CKD severity codes +KIDNEY,HCC-134,HCC-137,PLACEHOLDER +KIDNEY,HCC-134,HCC-138,PLACEHOLDER +KIDNEY,HCC-136,HCC-137,PLACEHOLDER +KIDNEY,HCC-136,HCC-138,PLACEHOLDER +KIDNEY,HCC-137,HCC-138,PLACEHOLDER +PULMONARY,HCC-111,HCC-112,PLACEHOLDER diff --git a/data/reference/hcc_v28_labels.csv b/data/reference/hcc_v28_labels.csv new file mode 100644 index 0000000000000000000000000000000000000000..c55c5be2e0a4875394dceba3d64008c6a12e6728 --- /dev/null +++ b/data/reference/hcc_v28_labels.csv @@ -0,0 +1,17 @@ +hcc_id,label,hierarchy_group,hierarchy_rank,raf_community_non_dual_aged,raf_community_dual_aged,raf_institutional,notes +HCC-17,Diabetes with Acute Complications,DIABETES,1,0.302,0.302,0.302,PLACEHOLDER seed RAF — replace via scripts/fetch_cms_hcc_v28.py +HCC-18,Diabetes with Chronic Complications,DIABETES,2,0.302,0.302,0.302,PLACEHOLDER +HCC-19,Diabetes without Complication,DIABETES,3,0.105,0.105,0.105,PLACEHOLDER +HCC-35,End-Stage Liver Disease,LIVER,1,1.025,1.025,1.025,PLACEHOLDER +HCC-37,Chronic Hepatitis,LIVER,2,0.225,0.225,0.225,PLACEHOLDER +HCC-48,Coagulation Defects and Other Specified Hematological Disorders,HEMATOLOGY,1,0.199,0.199,0.199,PLACEHOLDER +HCC-85,Congestive Heart Failure,CHF,1,0.333,0.333,0.333,PLACEHOLDER +HCC-88,Angina Pectoris,ISCHEMIC_HEART,1,0.127,0.127,0.127,PLACEHOLDER +HCC-96,Specified Heart Arrhythmias,HEART_ARRHYTHMIA,1,0.225,0.225,0.225,PLACEHOLDER +HCC-108,Vascular Disease,VASCULAR,1,0.288,0.288,0.288,PLACEHOLDER +HCC-111,Chronic Obstructive Pulmonary Disease,PULMONARY,1,0.335,0.335,0.335,PLACEHOLDER +HCC-112,Fibrosis of Lung and Other Chronic Lung Disorders,PULMONARY,2,0.221,0.221,0.221,PLACEHOLDER +HCC-134,Dialysis Status,KIDNEY,1,0.435,0.435,0.435,PLACEHOLDER +HCC-136,Chronic Kidney Disease Stage 5,KIDNEY,2,0.289,0.289,0.289,PLACEHOLDER +HCC-137,Chronic Kidney Disease Severe Stage 4,KIDNEY,3,0.289,0.289,0.289,PLACEHOLDER +HCC-138,Chronic Kidney Disease Moderate Stage 3,KIDNEY,4,0.070,0.070,0.070,PLACEHOLDER diff --git a/data/reference/icd10_cm_descriptions.csv b/data/reference/icd10_cm_descriptions.csv new file mode 100644 index 0000000000000000000000000000000000000000..8abf6f025546d4668ceb5837677cc5575884d222 --- /dev/null +++ b/data/reference/icd10_cm_descriptions.csv @@ -0,0 +1,55 @@ +icd10_code,description +E10.10,Type 1 diabetes mellitus with ketoacidosis without coma +E10.11,Type 1 diabetes mellitus with ketoacidosis with coma +E10.21,Type 1 diabetes mellitus with diabetic nephropathy +E10.22,Type 1 diabetes mellitus with diabetic chronic kidney disease +E10.40,Type 1 diabetes mellitus with diabetic neuropathy unspecified +E10.42,Type 1 diabetes mellitus with diabetic polyneuropathy +E10.9,Type 1 diabetes mellitus without complications +E11.00,Type 2 diabetes mellitus with hyperosmolarity without nonketotic hyperglycemic-hyperosmolar coma +E11.01,Type 2 diabetes mellitus with hyperosmolarity with coma +E11.21,Type 2 diabetes mellitus with diabetic nephropathy +E11.22,Type 2 diabetes mellitus with diabetic chronic kidney disease +E11.40,Type 2 diabetes mellitus with diabetic neuropathy unspecified +E11.42,Type 2 diabetes mellitus with diabetic polyneuropathy +E11.65,Type 2 diabetes mellitus with hyperglycemia +E11.9,Type 2 diabetes mellitus without complications +K70.30,Alcoholic cirrhosis of liver without ascites +K70.31,Alcoholic cirrhosis of liver with ascites +K74.60,Unspecified cirrhosis of liver +K74.69,Other cirrhosis of liver +B18.0,Chronic viral hepatitis B with delta-agent +B18.1,Chronic viral hepatitis B without delta-agent +B18.2,Chronic viral hepatitis C +D68.32,Hemorrhagic disorder due to extrinsic circulating anticoagulants +D68.4,Acquired coagulation factor deficiency +I50.9,Heart failure unspecified +I50.20,Unspecified systolic (congestive) heart failure +I50.22,Chronic systolic (congestive) heart failure +I50.30,Unspecified diastolic (congestive) heart failure +I50.32,Chronic diastolic (congestive) heart failure +I50.40,Unspecified combined systolic and diastolic (congestive) heart failure +I50.42,Chronic combined systolic and diastolic (congestive) heart failure +I20.0,Unstable angina +I20.1,Angina pectoris with documented spasm +I20.8,Other forms of angina pectoris +I20.9,Angina pectoris unspecified +I48.0,Paroxysmal atrial fibrillation +I48.1,Persistent atrial fibrillation +I48.2,Chronic atrial fibrillation +I48.91,Unspecified atrial fibrillation +I70.209,Unspecified atherosclerosis of native arteries of extremities +I70.219,Atherosclerosis of native arteries of extremities with intermittent claudication +I73.9,Peripheral vascular disease unspecified +J44.0,Chronic obstructive pulmonary disease with acute lower respiratory infection +J44.1,Chronic obstructive pulmonary disease with acute exacerbation +J44.9,Chronic obstructive pulmonary disease unspecified +J84.10,Pulmonary fibrosis unspecified +J84.112,Idiopathic pulmonary fibrosis +N18.30,Chronic kidney disease stage 3 unspecified +N18.31,Chronic kidney disease stage 3a +N18.32,Chronic kidney disease stage 3b +N18.4,Chronic kidney disease stage 4 (severe) +N18.5,Chronic kidney disease stage 5 +N18.6,End stage renal disease +Z99.2,Dependence on renal dialysis diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000000000000000000000000000000000000..78cb90c17d2327b7fd21057a237f129984f539e0 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,100 @@ +# Architecture + +## Data contract (single source of truth) + +[`app/pipeline/types.py`](../app/pipeline/types.py) is the authority. Every +component consumes and emits instances of these pydantic models. If you need +to add a field, add it here first and the components will fail loudly until +they handle it. + +Key shapes: +- `Note` — free-text input + optional section list. +- `Span` — typed offset ranges (Condition/Medication/Lab/...) with negation + temporality. +- `ICD10Link` — (Condition span, ICD-10 code, linker confidence). +- `HCCCandidate` — HCC id, RAF weight, hierarchy state, ICD-10 links, confidence (raw + calibrated), MEAT evidence, risk. +- `MEATEvidence` — category (M/E/A/T), quoted text + offsets, NLI entailment, `verified` flag. +- `RiskAssessment` — level (low/medium/high), score, transparent reason list, named component scores. +- `CodingResult` — the whole thing, serializable. + +## Component map + +| # | Component | Inputs | Outputs | Real or stub | +|---|---|---|---|---| +| A | Section Segmenter ([a_segmenter.py](../app/pipeline/components/a_segmenter.py)) | Note | Note with sections | Rule-based (real, narrow) | +| B | Clinical NER ([b_ner.py](../app/pipeline/components/b_ner.py)) | Note | `list[Span]` | Dictionary baseline (stub) | +| C | Negation + Temporality ([c_negation.py](../app/pipeline/components/c_negation.py)) | Note + spans | Spans with flags; `is_attributable()` filter | Rule-based (real) | +| D | ICD-10 Linker ([d_linker.py](../app/pipeline/components/d_linker.py)) | Spans | `list[ICD10Link]` | Exact + alias match (stub) | +| E | HCC Crosswalk ([e_crosswalk.py](../app/pipeline/components/e_crosswalk.py)) | Links | HCC candidates + trumping | **Real** (DuckDB + CSV) | +| F | MEAT Extractor ([f_meat.py](../app/pipeline/components/f_meat.py)) | Note + candidate | `list[MEATEvidence]` | Rule-based (stub) | +| G | Calibrator ([g_calibrator.py](../app/pipeline/components/g_calibrator.py)) | HCC + raw conf | Calibrated conf | Identity (stub) | +| H | Over-coding Risk ([h_risk.py](../app/pipeline/components/h_risk.py)) | Candidate | `RiskAssessment` | **Real** (transparent heuristic) | +| I | Verifier ([i_verifier.py](../app/pipeline/components/i_verifier.py)) | Note + candidate | Candidate with verified/dropped evidence | **Real** (substring) + NLI hook | + +## Orchestration order + +``` +A segmenter + → B NER + → C negation/temporality → filter non-attributable spans + → D linker (conditions only) + → E crosswalk + trumping + ↘ + F MEAT extractor (per candidate) + → I verifier (per candidate) + ↙ + G calibrator (per candidate) + → H risk (per candidate) + → rank: non-trumped first, then calibrated_conf × RAF_weight +``` + +Notes: +- I (verifier) sits **between F and G** — a candidate with zero verified MEAT + evidence is still kept (with an empty evidence list), because risk heuristic H + needs to see that emptiness and flag the candidate accordingly. Dropping the + candidate silently would undercount the suspect list a coder needs to see. +- G (calibrator) runs late because it doesn't affect rank tie-breakers inside E, + but it does affect the final ranking H reads. + +## Why hierarchy trumping is handled inside component E + +Trumping requires the full candidate set; it's not a per-span decision. Running +it inside the crosswalk (`HCCCrosswalk.apply_trumping`) keeps the orchestrator +code short and the crosswalk tests self-contained. + +## Why MEAT evidence belongs on HCCCandidate (not on Span) + +One MEAT evidence row justifies one HCC. The same sentence can justify Assess +for HCC-18 and Monitor for HCC-85. Binding MEATEvidence to HCC rather than to +span avoids a many-to-many association table. + +## API shape + +Endpoints in `app/api/server.py`: + +- `POST /api/code` — `{text, note_id}` → `CodingResult`. Single source of truth + for the frontend. +- `GET /api/gallery` — returns precomputed `GalleryItem[]` for the preloaded + demo cases. Computed once at app startup so clicking a gallery entry is instant. +- `GET /api/healthz` — `{status: "ok", version}`. Used by the Docker healthcheck. +- `GET /api/reference/hcc` — HCC label table for UI tooltips linking to CMS. + +## Frontend + +`ui/src/App.tsx` is a controller holding `noteText`, `result`, `selectedHCC`, +and the gallery. Three presentational components: +- `NotePanel` — textarea + highlight preview (MEAT quotes underlined by category). +- `HCCList` — ranked HCC suggestions; trumped ones dimmed. +- `DrillIn` — selected HCC's MEAT-by-category, risk reasons, ICD-10 support. + +State flows strictly one-way. The only network call besides gallery is +`codeNote()` from `ui/src/lib/api.ts`. + +## Deployment target + +Single Docker image (`Dockerfile`) for an HF **Docker Space**: +- Stage 1 (node:20-alpine): build the React UI. +- Stage 2 (python:3.11-slim): install minimal Python deps, copy `app/`, + `eval_harness/`, `data/reference/`, and the built UI at `ui/dist/`. +- Uvicorn serves FastAPI on :7860 and the UI via `StaticFiles` mount. +- HF Spaces auto-builds the Dockerfile; the Python CI runs on GitHub Actions + without building the image. diff --git a/docs/BUILD_LOG.md b/docs/BUILD_LOG.md new file mode 100644 index 0000000000000000000000000000000000000000..7c9281ec745846acee107fed44106f20c9b8617d --- /dev/null +++ b/docs/BUILD_LOG.md @@ -0,0 +1,157 @@ +# Build log + +Running record of what's been shipped, what's stubbed, and what's next. This +is the canonical "where does the project stand today" document. + +Every entry is written so that Charles (or a future agent) can audit it: + 1. **What changed** (files, scope) + 2. **Why** (brief-spec or trade-off) + 3. **How to verify** (commands, tests, UI actions) + 4. **What's still stubbed** (explicit) + +--- + +## 2026-04-23 — Milestone 0.1.0 "initial scaffold" + +### What changed + +**Repo bootstrap** +- `git init`, `.gitignore`, `pyproject.toml` (Python 3.11 + uv), `ruff` + `pytest` + config, `.pre-commit-config.yaml`, GitHub Actions CI at `.github/workflows/ci.yml`. +- Monorepo layout: `app/` (Python), `ui/` (React), `eval_harness/`, `tests/`, + `data/reference/`, `scripts/`, `docs/`. + +**Pipeline (Components A–I)** — `app/pipeline/` +- `types.py` — pydantic data contract (Note, Span, ICD10Link, HCCCandidate, + MEATEvidence, RiskAssessment, CodingResult). +- `components/a_segmenter.py` — rule-based SOAP/H&P header matcher. +- `components/b_ner.py` — `DictionaryNER` baseline (regex over a seeded vocabulary). +- `components/c_negation.py` — negation + family-history + temporality rules; + `is_attributable()` filter applied before HCC attribution. +- `components/d_linker.py` — `ExactMatchLinker` (aliases + description match; no embeddings yet). +- `components/e_crosswalk.py` — **real** DuckDB-backed lookup, builds HCC candidates, + applies hierarchy trumping per the CMS v28 hierarchy file + rank ordering. +- `components/f_meat.py` — `RuleBasedMEATExtractor` (sentence-level keyword cues; + substring-safe by construction). +- `components/g_calibrator.py` — `IdentityCalibrator` default + `IsotonicCalibrator` shell. +- `components/h_risk.py` — **real** over-coding risk heuristic (5 transparent components, + thresholds `low ≥ 0.70`, `medium ≥ 0.45`, `high` otherwise). +- `components/i_verifier.py` — **real** faithfulness verifier: substring guarantee + (non-negotiable) + NLI entailment hook (model injectable). +- `orchestrator.py` — wires A→B→C→D→E→F→I→G→H; ranks by (not trumped, then + `confidence_calibrated × max(raf_weight, 0.01)`). + +**API + CLI** — `app/api/`, `app/cli.py` +- FastAPI app with `/api/code`, `/api/gallery`, `/api/healthz`, `/api/reference/hcc`. +- Gallery is precomputed once on startup for instant first-click. +- `hcc-meat` Typer CLI: `code`, `gallery`, `serve`. + +**UI** — `ui/` +- Vite + TypeScript + Tailwind, React 18, three-panel layout. +- `App.tsx` orchestrates; `NotePanel` edits + highlights MEAT quotes; + `HCCList` shows ranked suggestions with risk + MEAT coverage pills; + `DrillIn` renders MEAT-by-category, risk reasons, ICD-10 evidence. +- `DisclaimerBanner` always-on amber bar at the top. +- Dev proxy (`vite.config.ts`) forwards `/api` to FastAPI :7860. + +**Reference data** — `data/reference/` +- `hcc_v28_labels.csv` — 16 seed HCCs with **PLACEHOLDER** RAF weights. +- `hcc_v28_crosswalk.csv` — ~50 ICD-10 → HCC mappings. +- `hcc_v28_hierarchy.csv` — trumping edges for DIABETES, LIVER, KIDNEY, PULMONARY groups. +- `icd10_cm_descriptions.csv` — descriptions for every code used in the crosswalk. +- `PROVENANCE.md` — license + how to hydrate from CMS.gov via `scripts/fetch_cms_hcc_v28.py`. + +**Regression fixtures** — `tests/fixtures/` +- `schema.py` — pydantic fixture schema (FixtureCase, ExpectedHCC). +- `regression_suite.json` — 10 cases covering: easy DM2, family history, + negation, historical chronic, DM hierarchy trumping, CHF insufficient-MEAT, + multi-specialty, short-note segmenter fallback, dialysis-over-CKD trumping, + synonym robustness. + +**Tests** — `tests/` +- `test_reference_data.py` — crosswalk referential integrity. +- `test_crosswalk.py` — HCC lookup, grouping, trumping across and within groups. +- `test_risk_heuristic.py` — level thresholds, reasons, component scores. +- `test_verifier.py` — substring check, fabrication drop, NLI gating. +- `test_segmenter_and_negation.py` — headers, family-history section, historical + chronic keep rule. +- `test_regression_suite.py` — runs the 10-case fixture through the pipeline. + +**Docker** — `Dockerfile`, `.dockerignore` +- Two-stage build: Node 20 for UI build → Python 3.11 slim runtime. +- Health check on `/api/healthz`. `EXPOSE 7860` (HF Spaces default). + +**Docs** — `README.md`, `docs/BUILD_LOG.md` (this), `docs/EVALUATION.md`, +`docs/ARCHITECTURE.md`, `docs/MODEL_CARD_TEMPLATE.md`, `docs/CLAUDE.md`. + +### Why + +Brief sections 1–16 locked by Charles on 2026-04-23: +- Section 16 answers: M4-Max local dev, public MEAT dataset, best-effort BYO-note, + payment HCCs first, MedGemma demo-only, single repo + dual remote, + 1.2/1.3 kept separate, React-in-Docker-Space UI. +- S1 (faithfulness) and S2 (CPU latency) are front-of-mind: substring guarantee + is compiled into Component I on day one; rule-based baselines make the whole + pipeline CPU-only until ML swap-ins land in Week 2. + +### How to verify + +```bash +# Static checks +uv run ruff check . +uv run ruff format --check . + +# Unit + fixture tests (ML extras not required) +uv run pytest -m "not ml" + +# CLI sanity +uv run hcc-meat code tests/fixtures/regression_suite.json # (won't parse JSON as plain text — that's expected) +echo "HPI: 67yo with DM2, HbA1c 9.2, on metformin." > /tmp/n.txt +uv run hcc-meat code /tmp/n.txt + +# Regression report +uv run hcc-meat-eval regression --json-out /tmp/regression.json + +# Backend + UI +uv run hcc-meat serve & +(cd ui && npm install && npm run dev) +# → browse http://localhost:5173 +# → click "Load example ▾" → "Straightforward DM2 with chronic complications" +# → verify HCC-18 appears first, HCC-19 absent or trumped + +# Docker image build (for the HF Space) +docker build -t hcc-meat-coder:0.1.0 . +docker run --rm -p 7860:7860 hcc-meat-coder:0.1.0 +# → curl http://localhost:7860/api/healthz +# → open http://localhost:7860 (UI is served from /ui/dist) +``` + +### What's still stubbed (explicit) + +| Area | Stub | Unblock path | +|---|---|---| +| RAF weights in `hcc_v28_labels.csv` | Placeholder numbers | `scripts/fetch_cms_hcc_v28.py` needs implementation | +| Component B NER | Regex dictionary | Swap to `blaze999/Medical-NER` or OpenMed NER; keep dictionary as CI fallback | +| Component D linker | Exact + alias match | SapBERT/MedCPT bi-encoder + cross-encoder re-rank | +| Component F MEAT extractor | Keyword-cue sentence picker | Phi-4-mini or Qwen2.5-3B with Outlines-constrained JSON output | +| Component G calibrator | Identity pass-through | `scripts/train_calibrator.py` (TBD) on held-out Synthea | +| Component I NLI check | `None` → trivially "verified" | Wire `cross-encoder/nli-deberta-v3-base` | +| Synthea generator | Not downloaded | `scripts/run_synthea.sh` (TBD) | +| MultiCaRe ingestion | Not downloaded | `scripts/ingest_multicare.py` (TBD) | +| MEAT dataset publication | Not generated | Week 3 | +| Reliability / subgroup tabs in UI | Routes present, no data | Week 4–5 | + +### Next milestone target (2026-04-30) + +Week 1 close: CMS data hydrated, Synthea generator wired, dictionary NER replaced +with the real Transformers NER model behind a feature flag, CI green on CPU tier. + +--- + +## Backlog / known questions + +- M4 Max unified memory capacity still unspecified — will bias model choice + (Phi-4-mini if 36 GB, Qwen-2.5-3B or MedGemma if 48+ GB). +- HF remote setup — waiting on access token from Charles. +- BYO-note PHI scanner: `presidio-analyzer` vs lightweight regex pass TBD. +- Non-payment HCCs: deferred to a stretch pass after payment HCCs land. diff --git a/docs/CLAUDE.md b/docs/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..fc574139af373a3e3450d4c98c68c9ec84c81eec --- /dev/null +++ b/docs/CLAUDE.md @@ -0,0 +1,59 @@ +# CLAUDE.md + +Guidance for any agent (me, a future agent, or another Claude session) picking +up this project. + +## The style bar +From the brief and from Charles on 2026-04-23: +- **Calibrated + auditable > peak F1.** If a choice trades 2 points of F1 for + interpretability or audit-defensibility, take the interpretability. +- **Clinical + coder vocabulary** in UI copy and docs (RAF, HCC capture, suspect + list, RADV-defensibility, MEAT-compliance, hierarchy trumping). Not ML-researcher voice. +- **Disclaimers always on, never ironic.** The amber banner at the top of the UI + does not get moved, hidden, or softened. + +## Operating posture + +Charles wants autonomous execution on this project. Translate that as: +- Build and review continuously. Document what changed in `docs/BUILD_LOG.md`. +- Scope-level and design-level decisions still need his approval — flag trade-offs, + don't silently resolve them. +- Trivia (naming, test layout, which logger) is mine to decide. + +## Invariants that must never regress + +1. **Faithfulness (S1).** Every MEAT quote is a byte-exact substring of the + input note. Component I is the enforcer; the invariant is tested in + `test_verifier.py` and `test_regression_suite.py::test_result_is_faithful`. +2. **Negation + family-history filter.** Anything marked negated, family-history, + or hypothetical is NOT attributed. Enforced by `c_negation.is_attributable()` + in the orchestrator. Tested in `test_segmenter_and_negation.py`. +3. **Hierarchy trumping.** Within a group, the higher-rank HCC trumps lower ones. + Enforced in `e_crosswalk.apply_trumping()`. Tested in `test_crosswalk.py`. +4. **Over-coding risk is transparent.** Every `RiskAssessment` must carry a list + of reasons; the UI must display them. Weights are named constants in + `h_risk.py`, not magic numbers sprinkled through. +5. **CPU-friendly baseline.** The pipeline runs end-to-end without ML extras + (no torch, no transformers). Swap-ins add capability; they don't become + required dependencies for CI. + +## Before touching a component + +Read `docs/ARCHITECTURE.md`, then the component file. Then `AGENT_BRIEF.md` §8 +for the component's design constraints. Then write a failing test in `tests/`, +then make it pass. Do not ship components that aren't covered by a regression +fixture case. + +## When unsure + +Ask Charles. The Section 16 handoff answers on 2026-04-23 are recorded in +memory (`MEMORY.md`); honor them until he says otherwise. Unresolved open +questions live in `docs/BUILD_LOG.md` under "Backlog / known questions". + +## Things explicitly to skip + +- Do not add dependencies on MIMIC-IV, DrugBank, UMLS licensed content, or any + PhysioNet-credentialed resource. +- Do not add MedGemma as a required dependency — it's demo-only per the license + conversation on 2026-04-23. +- Do not add a one-off amending commit after a failing hook — fix and commit new. diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md new file mode 100644 index 0000000000000000000000000000000000000000..b42ddf7af1856d45869ee95049c7daa0143660b0 --- /dev/null +++ b/docs/EVALUATION.md @@ -0,0 +1,107 @@ +# Evaluation guide + +How Charles (or any reviewer) can convince themselves the project works, and +how to interpret the numbers once ML components are swapped in. + +## 1. The 60-second smoke test + +```bash +uv sync --extra dev +uv run pytest -m "not ml" +uv run hcc-meat-eval regression +uv run hcc-meat-eval faithfulness +``` + +Expected: every unit test green; `regression` reports 10/10 PASS (or it lists +the exact fixture names and what was missing/forbidden); `faithfulness` prints +`0 fabrications`. + +If any of those fails on a fresh checkout, fix before anything else. + +## 2. Interactive UI walkthrough (5 minutes) + +```bash +uv run hcc-meat serve & +(cd ui && npm install && npm run dev) +``` + +Open http://localhost:5173 and click through the ten gallery items. For each, +verify: + +| Case | What to look for | +|---|---| +| Straightforward DM2 with chronic complications | HCC-18 appears first, LOW risk, all four MEAT cells filled | +| Family history of malignancy | **No HCCs** in the list (empty accepted set) | +| Ruled-out diagnosis | No HCCs; "denies diabetes" does not code | +| Historical chronic DM | HCC-19 appears with reasonable MEAT coverage | +| DM hierarchy trumping | HCC-18 appears non-trumped; HCC-19 appears trumped-by HCC-18 (dimmed in the list) | +| CHF insufficient MEAT | HCC-85 appears but with MEDIUM risk; drill-in shows "Missing MEAT categories: Monitor, Evaluate, Treat" | +| Multi-specialty | At least HCC-85, HCC-137, HCC-18 captured | +| Short note | HCC-111 captured despite absence of headers | +| Dialysis trumps CKD | HCC-134 appears non-trumped; HCC-136 dimmed, labeled "trumped by HCC-134" | +| Synonym robustness | DM2 and CHF surface forms resolve to HCC-19 and HCC-85 | + +Every verified MEAT quote in the drill-in must visibly highlight in the note +preview on the left (color-coded by M/E/A/T). + +## 3. Metrics Charles should care about + +Priority matches the brief's success criteria. + +### S1 — Faithfulness (non-negotiable) +- **Substring rate:** 100%. Any deviation is a bug. +- **NLI entailment rate:** measured once the NLI model is wired. Target ≥ 0.8 on + the held-out MEAT dataset. +- **Verifier drop count:** reported per request in `CodingResult.verifier_drops` + and shown in the UI footer. Should stay near 0 once the LLM extractor is in use. + +### S2 — Latency (free CPU tier target < 8s per note) +- Measured via `CodingResult.latency_ms`. The rule-based baseline runs in < 50 ms + on any note. The LLM extractor is the bottleneck; plan to quantize to Q4 GGUF + and cache gallery outputs server-side. + +### S3 — Subgroup slices (UI surface) +- Not yet in the UI. Planned tabs: by age band, note length bucket, specialty, + common-vs-rare HCC. `eval_harness/cli.py subgroups` will emit the JSON that + feeds the tab. + +### S5 — Coding quality (aspirational, held-out) +- Macro-F1 on HCC code suggestion. +- Per-HCC precision / recall / F1 bar chart. +- Mean RAF captured vs ground-truth RAF. +- Reported via `eval_harness/cli.py` once a labeled Synthea held-out set exists. + +### Calibration +- Reliability diagram (10 bins). +- Brier score, ECE. +- Only meaningful once the calibrator is fit, not while it's an identity pass-through. + +### Over-coding audit (built-in because the brief asks for it) +- For each candidate the pipeline marks LOW risk, does the held-out label confirm + MEAT actually supports it? Reports `audit_false_positive_rate`. + +## 4. Where the numbers come from + +`eval_harness/cli.py` subcommands: + +| Subcommand | What it does | Status | +|---|---|---| +| `regression` | Run the fixture suite, emit pass/fail report | ✅ Implemented | +| `faithfulness` | Assert every MEAT quote is a byte-exact substring of the source | ✅ Implemented | +| `subgroups` | Sliced metrics (age, length, specialty, frequency) | Stub | +| `calibration` | Reliability diagram + Brier + ECE | Stub | + +Run any subcommand with `--help` for options. + +## 5. What "good enough to ship v1" looks like + +From `AGENT_BRIEF.md` §3 in priority order: +- S1 faithfulness — 100% substring, ≥ 0.8 NLI entailment. +- S2 latency — p95 < 8s on free CPU tier with the gallery precomputed. +- S3 subgroups — rendered in the UI, not hidden in a notebook. +- S4 30-second recruiter skim — clear clinical value without ML jargon. +- S5 macro-F1 > 0.55 on held-out HCC suggestion, OR an honest report of a lower + number with a clear explanation in the model card. + +If we undershoot S5, ship with the honest number rather than extending the +timeline — the portfolio signal is **auditable + calibrated**, not SOTA F1. diff --git a/docs/MODEL_CARD_TEMPLATE.md b/docs/MODEL_CARD_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..f6a27781c1e04695af19eff4051d6e1b1090c1e8 --- /dev/null +++ b/docs/MODEL_CARD_TEMPLATE.md @@ -0,0 +1,48 @@ +# Model card — HCC-MEAT Coder (template) + +_Placeholder. Populate once the first ML components are trained/selected and the +held-out metrics are computed. Do not publish the Space without this filled in._ + +## Intended use +- **Primary:** portfolio / research demo of auditable HCC code suggestion. +- **Secondary:** teaching material for MEAT-compliance literacy. + +## Out-of-scope use +- Real clinical coding, billing, or claim submission. +- Any input containing PHI. +- ICD-10-PCS (procedures) or CPT E/M coding. + +## Data +- **Training:** Synthea (Apache 2.0) synthetic notes; LLM-annotated MEAT spans on + MultiCaRe (permissive per case). +- **Evaluation:** held-out Synthea + LLM-gold MultiCaRe split. +- **Excluded:** MIMIC-IV-Note (PhysioNet credentialed — not redistributable). + +## Metrics +_Fill from `eval_harness/cli.py` output._ +- Macro-F1 on HCC suggestion: TBD +- Per-HCC P/R/F1: TBD +- MEAT span exact-match F1: TBD +- Faithfulness: substring rate TBD, NLI entailment rate TBD +- Calibration: Brier TBD, ECE TBD +- Subgroup slices: TBD + +## Known failure modes +- Family-history mentions inside free prose without the section header (C is + rule-based so it relies on surface cues). +- Non-English notes (not evaluated). +- Very short notes (< 50 characters) often lack enough context for the MEAT + extractor. +- The v28 RAF weights are CMS-published values — if we ship while the + `data/reference/` tables still contain `PLACEHOLDER` rows, all RAF-scaled + numbers in the UI are invalid. Block release on this. + +## Licenses of upstream components +- CMS-HCC v28 crosswalk: public domain (US gov work). +- ICD-10-CM descriptions: public domain. +- Synthea: Apache 2.0. +- MultiCaRe cases: per-paper CC license, audited per case. +- NER / LLM / NLI weights: filled in when selected. + +## Feedback +Open an issue at `github.com/chaaarlieyap/hcc-meat-coder` (repo pending). diff --git a/eval_harness/__init__.py b/eval_harness/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..34d651bd9cc3322cab40afb26b5a82b9386936cb --- /dev/null +++ b/eval_harness/__init__.py @@ -0,0 +1,7 @@ +"""Evaluation harness for the HCC-MEAT coder. + +Importable as a package; also exposes a CLI via `hcc-meat-eval`. +See `docs/EVALUATION.md` for usage + metric definitions. +""" + +__version__ = "0.1.0" diff --git a/eval_harness/cli.py b/eval_harness/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..13b6fc7fe5618c2e00ac1c10a60a811d8dabd733 --- /dev/null +++ b/eval_harness/cli.py @@ -0,0 +1,113 @@ +"""`hcc-meat-eval` CLI — regression and metrics reporting. + +Subcommands (implemented / stubbed): + regression — run the fixture suite, emit a markdown report. + faithfulness — check that every MEAT quote is a substring of its source. + subgroups — sliced metrics (stub). + calibration — reliability diagram (stub). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import typer +from rich.console import Console +from rich.table import Table + +from app.pipeline import Pipeline +from app.pipeline.types import Note +from tests.fixtures.schema import load_suite + +app = typer.Typer(help=__doc__, no_args_is_help=True) +console = Console() + + +@app.command() +def regression( + suite: Path = typer.Option(Path("tests/fixtures/regression_suite.json")), + json_out: Path | None = typer.Option(None, help="Write machine-readable JSON report here."), +) -> None: + """Run every fixture case and report hit/miss per expectation.""" + fs = load_suite(suite) + pipe = Pipeline() + table = Table(title="Regression suite") + table.add_column("Case") + table.add_column("Tags") + table.add_column("Expected") + table.add_column("Got") + table.add_column("Status") + + report = [] + for case in fs.cases: + result = pipe.run(Note(note_id=case.case_id, text=case.note_text)) + got = {c.hcc for c in result.accepted()} + expected = {e.hcc_id for e in case.expected_hccs if e.must_appear} + forbidden = set(case.must_not_appear_hccs) + missing = expected - got + extra = got & forbidden + ok = not missing and not extra + status = "PASS" if ok else "FAIL" + report.append({ + "case_id": case.case_id, + "status": status, + "missing": sorted(missing), + "forbidden_present": sorted(extra), + "got": sorted(got), + }) + table.add_row( + case.case_id, + ", ".join(case.tags), + ", ".join(sorted(expected)) or "—", + ", ".join(sorted(got)) or "—", + f"[green]{status}[/green]" if ok else f"[red]{status}[/red]", + ) + + console.print(table) + passed = sum(1 for r in report if r["status"] == "PASS") + console.print(f"{passed}/{len(report)} passed") + + if json_out is not None: + json_out.write_text(json.dumps(report, indent=2)) + + +@app.command() +def faithfulness( + suite: Path = typer.Option(Path("tests/fixtures/regression_suite.json")), +) -> None: + """Assert every returned MEAT quote is an exact substring of its source note.""" + fs = load_suite(suite) + pipe = Pipeline() + failures: list[str] = [] + totals = 0 + for case in fs.cases: + r = pipe.run(Note(note_id=case.case_id, text=case.note_text)) + for c in r.candidates: + for ev in c.meat_evidence: + totals += 1 + if ev.quote not in case.note_text: + failures.append( + f"{case.case_id}:{c.hcc}:{ev.category.value} → {ev.quote!r}" + ) + if failures: + for f in failures: + console.print(f"[red]{f}[/red]") + raise typer.Exit(1) + console.print(f"[green]faithfulness OK[/green] ({totals} quotes, 0 fabrications)") + + +@app.command() +def subgroups() -> None: # stub + console.print("[yellow]subgroup slicing not implemented yet[/yellow]") + raise typer.Exit(1) + + +@app.command() +def calibration() -> None: # stub + console.print("[yellow]calibration reporting not implemented yet[/yellow]") + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..9f922fb45815cebdafc3b2cbb406ae564e43119b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[project] +name = "hcc-meat-coder" +version = "0.1.0" +description = "HCC (CMS-HCC v28) code suggester with MEAT-justified evidence, calibrated confidence, and over-coding risk flags." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "Apache-2.0" } +authors = [{ name = "Charles Yap", email = "chaaarlieyap@gmail.com" }] + +dependencies = [ + "fastapi>=0.115", + "uvicorn[standard]>=0.32", + "pydantic>=2.9", + "duckdb>=1.1", + "pandas>=2.2", + "numpy>=2.0", + "scikit-learn>=1.5", + "matplotlib>=3.9", + "typer>=0.12", + "rich>=13.9", + "httpx>=0.27", +] + +[project.optional-dependencies] +ml = [ + "transformers>=4.46", + "sentence-transformers>=3.2", + "torch>=2.5", + "outlines>=0.1", + "medspacy>=1.3", + "spacy>=3.7", +] +llama = [ + "llama-cpp-python>=0.3", +] +dev = [ + "pytest>=8.3", + "pytest-cov>=5.0", + "ruff>=0.7", + "pre-commit>=4.0", + "ipykernel>=6.29", + "jupyter>=1.1", +] + +[project.scripts] +hcc-meat = "app.cli:app" +hcc-meat-eval = "eval_harness.cli:app" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["app", "eval_harness"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"] +ignore = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = "test_*.py" +addopts = "-ra --strict-markers" +markers = [ + "slow: marks tests that take longer than 1s", + "ml: marks tests that require ML dependencies installed", +] diff --git a/scripts/fetch_cms_hcc_v28.py b/scripts/fetch_cms_hcc_v28.py new file mode 100644 index 0000000000000000000000000000000000000000..f49650aa35c869fb337dc79c2c607675081b4012 --- /dev/null +++ b/scripts/fetch_cms_hcc_v28.py @@ -0,0 +1,49 @@ +"""Fetch the CMS-HCC v28 model tables and ICD-10-CM descriptions into data/reference/. + +Run from repo root: + uv run python scripts/fetch_cms_hcc_v28.py --out data/reference/ + +This replaces the placeholder seed tables with the official public CMS / CDC values. + +Sources: + - CMS Risk Adjustment landing page: + https://www.cms.gov/medicare/payment/medicare-advantage-rates-statistics/risk-adjustment + The v28 model ZIP contains the HCC label table, ICD-10 to HCC crosswalk, + hierarchy file, and the RAF coefficients (Table V-1 of the Advance Notice). + - CDC NCHS ICD-10-CM annual release: + https://www.cdc.gov/nchs/icd/icd-10-cm.htm + +Until this script is implemented end-to-end, the seed files shipped in +`data/reference/` are intentionally marked PLACEHOLDER in their row comments. The +pipeline runs against them for development and tests; metrics published externally +must use fetched data. + +TODO: + 1. Locate and parse the v28 model ZIP (CMS changes the file name per release). + 2. Parse ICD-10-CM descriptor file (fixed-width or CSV depending on year). + 3. Normalize column names to match the seed CSV schema. + 4. Emit a `PROVENANCE.lock` next to the CSVs capturing source URLs + SHA-256. +""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +app = typer.Typer(help=__doc__, no_args_is_help=True) + + +@app.command() +def main( + out: Path = typer.Option(Path("data/reference"), help="Output directory."), + year: int = typer.Option(2024, help="CMS model year for v28 tables."), +) -> None: + """Hydrate data/reference/ from public CMS + CDC sources.""" + raise typer.Exit( + code=1, + ) # Intentional: surfaces that this is not yet implemented until first real run. + + +if __name__ == "__main__": + app() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/fixtures/regression_suite.json b/tests/fixtures/regression_suite.json new file mode 100644 index 0000000000000000000000000000000000000000..7c587f9881a6c7eb6a85e4e7e4b802a54fb169d2 --- /dev/null +++ b/tests/fixtures/regression_suite.json @@ -0,0 +1,132 @@ +{ + "version": "0.1.0", + "cases": [ + { + "case_id": "dm2-complications-simple", + "name": "Straightforward DM2 with chronic complications", + "tags": ["positive", "diabetes", "easy"], + "note_text": "CHIEF COMPLAINT: Follow-up diabetes.\n\nHPI: 67-year-old male with Type 2 diabetes mellitus with diabetic polyneuropathy. HbA1c 9.2 today, up from 8.4 three months ago. Reports burning pain in feet.\n\nPHYSICAL EXAM: Decreased sensation to monofilament bilaterally.\n\nASSESSMENT AND PLAN: Type 2 diabetes mellitus with diabetic polyneuropathy, uncontrolled. Continue metformin 1000 mg BID. Start insulin glargine 10 units nightly. Monitor HbA1c in 3 months. Referred to endocrinology.", + "expected_hccs": [ + { + "hcc_id": "HCC-18", + "must_appear": true, + "expected_risk_level": "low", + "expected_meat_categories": ["M", "E", "A", "T"] + } + ], + "must_not_appear_hccs": ["HCC-19"], + "commentary": "DM2 with chronic complications (HCC-18) should capture; DM2 without complication (HCC-19) must NOT appear since complication is present." + }, + { + "case_id": "family-history-only", + "name": "Family history of malignancy — must not code", + "tags": ["negative", "family_history"], + "note_text": "HPI: 55-year-old female here for routine visit. No acute complaints.\n\nFAMILY HISTORY: Father had colon cancer at age 62. Mother had Type 2 diabetes mellitus. Brother with congestive heart failure.\n\nASSESSMENT: Healthy. Continue current lifestyle measures.", + "expected_hccs": [], + "must_not_appear_hccs": ["HCC-17", "HCC-18", "HCC-19", "HCC-85"], + "commentary": "All condition mentions are in FAMILY HISTORY section. Must not code." + }, + { + "case_id": "negation-ruled-out", + "name": "Ruled-out diagnosis — must not code", + "tags": ["negative", "negation"], + "note_text": "HPI: 72-year-old presents with cough and fever. Chest X-ray without infiltrate.\n\nASSESSMENT: Viral URI. No evidence of heart failure. Ruled out pneumonia. Denies diabetes mellitus.\n\nPLAN: Supportive care. Follow up in one week.", + "expected_hccs": [], + "must_not_appear_hccs": ["HCC-19", "HCC-85", "HCC-111"], + "commentary": "All conditions negated (\"no evidence of\", \"ruled out\", \"denies\"). Must not code." + }, + { + "case_id": "historical-chronic-dm", + "name": "Historical mention of chronic DM — should still code", + "tags": ["positive", "temporality", "chronic"], + "note_text": "HPI: 70-year-old male with history of Type 2 diabetes mellitus. Reports good control on current regimen. HbA1c today 6.8.\n\nMEDICATIONS: Metformin 500 mg BID.\n\nASSESSMENT AND PLAN: Stable Type 2 diabetes mellitus. Continue metformin. Monitor HbA1c in 6 months.", + "expected_hccs": [ + { + "hcc_id": "HCC-19", + "must_appear": true, + "expected_meat_categories": ["M", "E", "A", "T"] + } + ], + "must_not_appear_hccs": ["HCC-18"], + "commentary": "\"History of\" a chronic disease is still codable when the note confirms ongoing management." + }, + { + "case_id": "trumping-dm", + "name": "DM w/o complications mentioned early, w/ neuropathy later — HCC-18 trumps HCC-19", + "tags": ["positive", "trumping", "diabetes"], + "note_text": "HPI: 64-year-old with Type 2 diabetes mellitus without complications by prior documentation. However on exam today, monofilament testing shows decreased sensation. HbA1c 8.1.\n\nASSESSMENT AND PLAN: Type 2 diabetes mellitus with diabetic polyneuropathy, new diagnosis. Titrated metformin to 1000 mg BID. Started gabapentin 300 mg nightly. Monitor A1c in 3 months.", + "expected_hccs": [ + { + "hcc_id": "HCC-18", + "must_appear": true, + "expected_meat_categories": ["E", "A", "T"] + }, + { + "hcc_id": "HCC-19", + "must_appear": true, + "must_be_trumped_by": "HCC-18" + } + ], + "commentary": "Both HCC-18 and HCC-19 may be linked from the ICD-10s, but HCC-18 (chronic complications) must trump HCC-19." + }, + { + "case_id": "chf-insufficient-meat", + "name": "CHF diagnosis in A/P but no Monitor or Evaluate evidence — must flag higher risk", + "tags": ["positive", "insufficient_meat", "risk_flag"], + "note_text": "ASSESSMENT AND PLAN: Congestive heart failure, diagnosed today. Plan for cardiology referral.", + "expected_hccs": [ + { + "hcc_id": "HCC-85", + "must_appear": true, + "expected_risk_level": "medium", + "expected_meat_categories": ["A"] + } + ], + "commentary": "Diagnosis is present but with sparse MEAT. The risk heuristic should not return LOW for this candidate." + }, + { + "case_id": "multi-specialty", + "name": "Multi-specialty note capturing CHF + CKD + DM", + "tags": ["positive", "multi-hcc"], + "note_text": "HPI: 78-year-old with known congestive heart failure, CKD stage 4, and Type 2 diabetes mellitus with diabetic nephropathy.\n\nLABS: Creatinine 2.8, eGFR 22, BNP 1450. HbA1c 8.9.\n\nASSESSMENT AND PLAN: 1) Chronic systolic (congestive) heart failure — continue carvedilol and furosemide, monitor daily weights. 2) Chronic kidney disease stage 4 (severe) — nephrology follow-up. 3) Type 2 diabetes mellitus with diabetic nephropathy — continue insulin glargine, titrate based on home glucose logs.", + "expected_hccs": [ + { "hcc_id": "HCC-85", "must_appear": true, "expected_meat_categories": ["M", "E", "A", "T"] }, + { "hcc_id": "HCC-137", "must_appear": true, "expected_meat_categories": ["E", "A", "T"] }, + { "hcc_id": "HCC-18", "must_appear": true, "expected_meat_categories": ["E", "A", "T"] } + ], + "commentary": "All three HCCs have MEAT coverage across independent sections." + }, + { + "case_id": "short-note", + "name": "Very short assessment-only note — stress tests the segmenter fallback", + "tags": ["positive", "short_note"], + "note_text": "A/P: Chronic obstructive pulmonary disease exacerbation. Continue albuterol and start prednisone 40 mg daily for 5 days.", + "expected_hccs": [ + { "hcc_id": "HCC-111", "must_appear": true, "expected_meat_categories": ["A", "T"] } + ], + "commentary": "Short free-text should still resolve to HCC-111 via alias matching." + }, + { + "case_id": "dialysis-trumps-ckd", + "name": "Dialysis status trumps CKD stage 5", + "tags": ["positive", "trumping", "kidney"], + "note_text": "HPI: 60-year-old on hemodialysis three times weekly for end stage renal disease with CKD stage 5.\n\nASSESSMENT AND PLAN: Dependence on renal dialysis — continue current HD schedule. Monitor monthly labs.", + "expected_hccs": [ + { "hcc_id": "HCC-134", "must_appear": true, "expected_meat_categories": ["M", "A", "T"] }, + { "hcc_id": "HCC-136", "must_appear": true, "must_be_trumped_by": "HCC-134" } + ], + "commentary": "HCC-134 (Dialysis Status) trumps HCC-136 (CKD Stage 5)." + }, + { + "case_id": "synonym-robustness", + "name": "Synonym variation — \"DM2\", \"a1c\", \"CHF\" surface forms", + "tags": ["positive", "robustness", "synonyms"], + "note_text": "HPI: DM2 on metformin. A1c 7.8. CHF stable, on carvedilol and lisinopril.\n\nASSESSMENT AND PLAN: Stable DM2 and CHF. Continue current meds. Monitor A1c in 3 months and repeat echocardiogram in 6.", + "expected_hccs": [ + { "hcc_id": "HCC-19", "must_appear": true, "expected_meat_categories": ["E", "A", "T"] }, + { "hcc_id": "HCC-85", "must_appear": true, "expected_meat_categories": ["E", "A", "T"] } + ], + "commentary": "The linker's alias table must resolve \"DM2\" and \"CHF\"." + } + ] +} diff --git a/tests/fixtures/schema.py b/tests/fixtures/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..58ddb8bd17275c22e448c1dd91874b3a6d3da923 --- /dev/null +++ b/tests/fixtures/schema.py @@ -0,0 +1,45 @@ +"""Regression-fixture schema. + +Each fixture is a clinical note with explicit expectations about what the +pipeline must — or must not — produce. Fixtures are the contract between the +deterministic parts of the pipeline (crosswalk, trumping, negation filter) and +the rest of the system; they run in CI on every push. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, Field + + +class ExpectedHCC(BaseModel): + hcc_id: str + must_appear: bool = True + must_be_trumped_by: str | None = None + min_confidence_raw: float | None = Field(default=None, ge=0.0, le=1.0) + expected_risk_level: Literal["low", "medium", "high"] | None = None + expected_meat_categories: list[Literal["M", "E", "A", "T"]] = Field(default_factory=list) + + +class FixtureCase(BaseModel): + case_id: str + name: str + tags: list[str] = Field(default_factory=list) + note_text: str + expected_hccs: list[ExpectedHCC] = Field(default_factory=list) + must_not_appear_hccs: list[str] = Field(default_factory=list) + commentary: str = "" + + +class FixtureSuite(BaseModel): + version: str = "0.1.0" + cases: list[FixtureCase] + + +def load_suite(path: Path) -> FixtureSuite: + import json + + with Path(path).open() as f: + return FixtureSuite.model_validate(json.load(f)) diff --git a/tests/test_crosswalk.py b/tests/test_crosswalk.py new file mode 100644 index 0000000000000000000000000000000000000000..6b1bda5ab4b6fd86b474c0024567c040094b2247 --- /dev/null +++ b/tests/test_crosswalk.py @@ -0,0 +1,83 @@ +"""Crosswalk + hierarchy trumping tests.""" + +from __future__ import annotations + +from app.pipeline.components.e_crosswalk import HCCCrosswalk +from app.pipeline.types import ICD10Link, Span, SpanType + + +def _make_link(icd10: str, text: str, conf: float = 0.8) -> ICD10Link: + return ICD10Link( + icd10=icd10, + description="", + linker_confidence=conf, + condition_span=Span( + start=0, + end=len(text), + text=text, + span_type=SpanType.CONDITION, + ner_confidence=0.8, + ), + ) + + +def test_icd10_to_hcc_lookup(): + cw = HCCCrosswalk() + assert cw.hcc_for_icd10("E11.42") == "HCC-18" + assert cw.hcc_for_icd10("E11.9") == "HCC-19" + assert cw.hcc_for_icd10("I50.22") == "HCC-85" + assert cw.hcc_for_icd10("Z99.2") == "HCC-134" + assert cw.hcc_for_icd10("FAKE.99") is None + + +def test_build_candidates_groups_links_by_hcc(): + cw = HCCCrosswalk() + links = [ + _make_link("E11.9", "diabetes"), + _make_link("E11.65", "diabetes with hyperglycemia", conf=0.7), + _make_link("I50.9", "heart failure"), + ] + cands = cw.build_candidates(links) + by_hcc = {c.hcc: c for c in cands} + assert "HCC-19" in by_hcc + assert "HCC-18" in by_hcc # E11.65 -> HCC-18 + assert "HCC-85" in by_hcc + assert len(by_hcc["HCC-19"].icd10_links) == 1 + + +def test_hierarchy_trumping_dm(): + cw = HCCCrosswalk() + links = [ + _make_link("E11.9", "diabetes"), # -> HCC-19 + _make_link("E11.42", "diabetic polyneuropathy"), # -> HCC-18 + ] + cands = cw.apply_trumping(cw.build_candidates(links)) + by_hcc = {c.hcc: c for c in cands} + assert by_hcc["HCC-19"].hierarchy.trumped is True + assert by_hcc["HCC-19"].hierarchy.trumped_by == "HCC-18" + assert by_hcc["HCC-18"].hierarchy.trumped is False + + +def test_hierarchy_trumping_dialysis_over_ckd(): + cw = HCCCrosswalk() + links = [ + _make_link("Z99.2", "dialysis"), # HCC-134 + _make_link("N18.5", "ckd stage 5"), # HCC-136 + _make_link("N18.4", "ckd stage 4"), # HCC-137 + ] + cands = cw.apply_trumping(cw.build_candidates(links)) + by_hcc = {c.hcc: c for c in cands} + assert by_hcc["HCC-134"].hierarchy.trumped is False + assert by_hcc["HCC-136"].hierarchy.trumped is True + assert by_hcc["HCC-137"].hierarchy.trumped is True + + +def test_hierarchy_isolation_across_groups(): + cw = HCCCrosswalk() + links = [ + _make_link("E11.42", "diabetic polyneuropathy"), # HCC-18 + _make_link("I50.9", "heart failure"), # HCC-85 + ] + cands = cw.apply_trumping(cw.build_candidates(links)) + for c in cands: + assert c.hierarchy.trumped is False, f"{c.hcc} was trumped across groups" diff --git a/tests/test_reference_data.py b/tests/test_reference_data.py new file mode 100644 index 0000000000000000000000000000000000000000..f4cb55967c4a403fc1563bc43c96ec8e27ef59a5 --- /dev/null +++ b/tests/test_reference_data.py @@ -0,0 +1,81 @@ +"""Sanity checks for the staged reference data. + +Guards against: + - Typo in an HCC id that appears only in one table. + - ICD-10 codes in the crosswalk that have no description. + - Hierarchy groups with no labeled members. +""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import pytest + +REF = Path("data/reference") + + +def _rows(path: Path) -> list[dict[str, str]]: + with path.open() as f: + return list(csv.DictReader(f)) + + +def test_all_crosswalk_hccs_are_labeled(): + labels = {r["hcc_id"] for r in _rows(REF / "hcc_v28_labels.csv")} + crosswalk_hccs = {r["hcc_id"] for r in _rows(REF / "hcc_v28_crosswalk.csv")} + missing = crosswalk_hccs - labels + assert not missing, f"HCCs in crosswalk but not in labels: {sorted(missing)}" + + +def test_all_crosswalk_icd10s_have_descriptions(): + desc = {r["icd10_code"] for r in _rows(REF / "icd10_cm_descriptions.csv")} + codes = {r["icd10_code"] for r in _rows(REF / "hcc_v28_crosswalk.csv")} + missing = codes - desc + assert not missing, f"ICD-10 codes in crosswalk but not in descriptions: {sorted(missing)}" + + +def test_hierarchy_references_known_hccs(): + labels = {r["hcc_id"] for r in _rows(REF / "hcc_v28_labels.csv")} + for r in _rows(REF / "hcc_v28_hierarchy.csv"): + assert r["higher_hcc"] in labels, f"Unknown higher_hcc: {r}" + assert r["trumps_lower_hcc"] in labels, f"Unknown trumps_lower_hcc: {r}" + + +def test_hierarchy_groups_are_consistent(): + groups_from_labels: dict[str, set[str]] = {} + for r in _rows(REF / "hcc_v28_labels.csv"): + groups_from_labels.setdefault(r["hierarchy_group"], set()).add(r["hcc_id"]) + + for r in _rows(REF / "hcc_v28_hierarchy.csv"): + higher = r["higher_hcc"] + lower = r["trumps_lower_hcc"] + # Both members of an explicit trump edge must be in the same hierarchy group. + higher_group = next( + (g for g, m in groups_from_labels.items() if higher in m), None + ) + lower_group = next( + (g for g, m in groups_from_labels.items() if lower in m), None + ) + assert higher_group == lower_group, ( + f"Trump edge crosses groups: {higher} ({higher_group}) → {lower} ({lower_group})" + ) + + +def test_raf_weights_are_non_negative(): + for r in _rows(REF / "hcc_v28_labels.csv"): + for col in ( + "raf_community_non_dual_aged", + "raf_community_dual_aged", + "raf_institutional", + ): + v = float(r[col]) + assert v >= 0.0, f"Negative RAF for {r['hcc_id']}.{col}: {v}" + + +@pytest.mark.parametrize( + "hcc_id", ["HCC-17", "HCC-18", "HCC-19", "HCC-85", "HCC-134", "HCC-111"] +) +def test_core_hccs_present(hcc_id): + labels = {r["hcc_id"] for r in _rows(REF / "hcc_v28_labels.csv")} + assert hcc_id in labels diff --git a/tests/test_regression_suite.py b/tests/test_regression_suite.py new file mode 100644 index 0000000000000000000000000000000000000000..04f4097ad901324e35e1f75e68b2428aa6ee550d --- /dev/null +++ b/tests/test_regression_suite.py @@ -0,0 +1,88 @@ +"""Run the pipeline against tests/fixtures/regression_suite.json and assert +expectations. These are the contract tests for the deterministic core — they +gate CI and are the first thing to inspect when a baseline regresses. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from app.pipeline import Pipeline +from app.pipeline.types import Note +from tests.fixtures.schema import FixtureCase, load_suite + +SUITE_PATH = Path(__file__).parent / "fixtures" / "regression_suite.json" + + +@pytest.fixture(scope="module") +def pipeline() -> Pipeline: + return Pipeline() + + +@pytest.fixture(scope="module") +def suite(): + return load_suite(SUITE_PATH) + + +def _run(pipeline: Pipeline, case: FixtureCase): + return pipeline.run(Note(note_id=case.case_id, text=case.note_text)) + + +@pytest.mark.parametrize("case_id", [ + "dm2-complications-simple", + "family-history-only", + "negation-ruled-out", + "historical-chronic-dm", + "trumping-dm", + "chf-insufficient-meat", + "multi-specialty", + "short-note", + "dialysis-trumps-ckd", + "synonym-robustness", +]) +def test_case(pipeline, suite, case_id): + case = next(c for c in suite.cases if c.case_id == case_id) + result = _run(pipeline, case) + got_hccs = {c.hcc for c in result.candidates} + + for expected in case.expected_hccs: + if expected.must_appear: + assert expected.hcc_id in got_hccs, ( + f"[{case.case_id}] expected HCC {expected.hcc_id} not found. " + f"Got: {sorted(got_hccs)}" + ) + cand = next(c for c in result.candidates if c.hcc == expected.hcc_id) + if expected.must_be_trumped_by is not None: + assert cand.hierarchy.trumped, ( + f"[{case.case_id}] {expected.hcc_id} should have been trumped" + ) + assert cand.hierarchy.trumped_by == expected.must_be_trumped_by, ( + f"[{case.case_id}] {expected.hcc_id} trumped by {cand.hierarchy.trumped_by}, " + f"expected {expected.must_be_trumped_by}" + ) + if expected.min_confidence_raw is not None: + assert cand.confidence_raw >= expected.min_confidence_raw + + for forbidden in case.must_not_appear_hccs: + # must_not_appear means: NOT present in accepted (non-trumped) list + accepted = {c.hcc for c in result.accepted()} + assert forbidden not in accepted, ( + f"[{case.case_id}] forbidden HCC {forbidden} was accepted. " + f"Accepted: {sorted(accepted)}" + ) + + +def test_result_is_faithful(pipeline, suite): + """Every MEAT quote returned must be an exact substring of the input note.""" + failures: list[str] = [] + for case in suite.cases: + result = _run(pipeline, case) + for cand in result.candidates: + for ev in cand.meat_evidence: + if ev.quote not in case.note_text: + failures.append( + f"{case.case_id}:{cand.hcc}:{ev.category.value} → {ev.quote!r}" + ) + assert not failures, "Fabricated MEAT quotes detected:\n" + "\n".join(failures) diff --git a/tests/test_risk_heuristic.py b/tests/test_risk_heuristic.py new file mode 100644 index 0000000000000000000000000000000000000000..52bed4b9550c06e2a625eefe2b098eccee93cd0e --- /dev/null +++ b/tests/test_risk_heuristic.py @@ -0,0 +1,115 @@ +"""Over-coding risk heuristic tests — weights, reasons, level thresholds.""" + +from __future__ import annotations + +from app.pipeline.components.h_risk import ( + LOW_THRESHOLD, + MEDIUM_THRESHOLD, + assess_risk, +) +from app.pipeline.types import ( + HCCCandidate, + HierarchyState, + ICD10Link, + MEATCategory, + MEATEvidence, + Span, + SpanType, +) + + +def _make_candidate( + *, + raf: float, + ner_conf: float, + linker_conf: float, + trumped: bool = False, + meat_filled: list[MEATCategory] | None = None, +) -> HCCCandidate: + condition_span = Span( + start=0, + end=10, + text="diabetes", + span_type=SpanType.CONDITION, + ner_confidence=ner_conf, + ) + link = ICD10Link( + icd10="E11.9", + description="dm", + linker_confidence=linker_conf, + condition_span=condition_span, + ) + meat_filled = meat_filled or [] + meat_evidence = [ + MEATEvidence( + category=cat, + quote="x", + start=0, + end=1, + extractor_confidence=0.5, + verified=True, + ) + for cat in meat_filled + ] + return HCCCandidate( + hcc="HCC-19", + hcc_label="Diabetes", + raf_weight=raf, + hierarchy=HierarchyState(group="DIABETES", rank=3, trumped=trumped, trumped_by="HCC-18" if trumped else None), + icd10_links=[link], + confidence_raw=ner_conf * linker_conf, + confidence_calibrated=ner_conf * linker_conf, + meat_evidence=meat_evidence, + ) + + +def test_low_risk_when_everything_is_strong(): + c = _make_candidate( + raf=0.10, + ner_conf=0.95, + linker_conf=0.90, + meat_filled=list(MEATCategory), + ) + r = assess_risk(c) + assert r.level == "low" + assert r.score >= LOW_THRESHOLD + + +def test_high_risk_when_meat_is_empty_and_raf_is_high(): + c = _make_candidate(raf=0.40, ner_conf=0.3, linker_conf=0.3, meat_filled=[]) + r = assess_risk(c) + assert r.level == "high" + assert r.score < MEDIUM_THRESHOLD + + +def test_medium_risk_for_partial_meat(): + c = _make_candidate( + raf=0.30, + ner_conf=0.7, + linker_conf=0.7, + meat_filled=[MEATCategory.ASSESS, MEATCategory.TREAT], + ) + r = assess_risk(c) + assert r.level in {"medium", "low"} # depends on exact weight sum + assert any("MEAT" in reason for reason in r.reasons) + + +def test_trumped_candidate_gets_hierarchy_reason(): + c = _make_candidate( + raf=0.10, + ner_conf=0.9, + linker_conf=0.9, + trumped=True, + meat_filled=list(MEATCategory), + ) + r = assess_risk(c) + assert any("Trumped" in reason for reason in r.reasons) + assert r.components["hierarchy_support"] == 0.0 + + +def test_reasons_are_human_readable(): + c = _make_candidate(raf=0.4, ner_conf=0.3, linker_conf=0.3, meat_filled=[]) + r = assess_risk(c) + for reason in r.reasons: + assert isinstance(reason, str) + assert len(reason) > 5 diff --git a/tests/test_segmenter_and_negation.py b/tests/test_segmenter_and_negation.py new file mode 100644 index 0000000000000000000000000000000000000000..2087ada9927a1f803de35c7e8c5c21c7ecb360c2 --- /dev/null +++ b/tests/test_segmenter_and_negation.py @@ -0,0 +1,85 @@ +"""Segmenter + negation filter tests.""" + +from __future__ import annotations + +from app.pipeline.components.a_segmenter import find_section, segment +from app.pipeline.components.c_negation import NegationTemporalityClassifier, is_attributable +from app.pipeline.types import Note, Span, SpanType, Temporality + + +def test_segmenter_splits_known_headers(): + note = Note( + note_id="n", + text=( + "CHIEF COMPLAINT:\nfollow-up diabetes\n\n" + "HPI:\nT2DM with complications\n\n" + "ASSESSMENT AND PLAN:\nT2DM with diabetic polyneuropathy." + ), + ) + note = segment(note) + kinds = [s.kind for s in note.sections] + assert "chief_complaint" in kinds + assert "hpi" in kinds + assert "assessment_and_plan" in kinds + + +def test_segmenter_without_headers_returns_single_other_section(): + note = Note(note_id="n", text="Patient has T2DM. Stable on metformin.") + note = segment(note) + assert len(note.sections) == 1 + assert note.sections[0].kind == "other" + + +def test_find_section_returns_correct_span(): + note = Note( + note_id="n", + text="HPI:\nT2DM\n\nASSESSMENT:\nStable T2DM.", + ) + note = segment(note) + ap = find_section(note, "assessment") + assert ap is not None + assert "Stable T2DM." in ap.text + + +def _make_span(note: Note, term: str) -> Span: + idx = note.text.lower().find(term.lower()) + return Span( + start=idx, + end=idx + len(term), + text=note.text[idx : idx + len(term)], + span_type=SpanType.CONDITION, + ner_confidence=0.8, + ) + + +def test_negation_flags_ruled_out(): + note = Note(note_id="n", text="Ruled out diabetes today. No heart failure.") + clf = NegationTemporalityClassifier() + spans = [_make_span(note, "diabetes"), _make_span(note, "heart failure")] + out = clf(note, spans) + assert all(s.negated for s in out) + assert not any(is_attributable(s) for s in out) + + +def test_family_history_section_marks_family_history(): + note = Note( + note_id="n", + text="FAMILY HISTORY:\nFather had diabetes. Mother had heart failure.", + ) + note = segment(note) + clf = NegationTemporalityClassifier() + spans = [_make_span(note, "diabetes"), _make_span(note, "heart failure")] + # attach section hint + out = clf(note, spans) + for s in out: + assert s.temporality == Temporality.FAMILY_HISTORY + assert not is_attributable(s) + + +def test_historical_chronic_condition_is_still_attributable(): + note = Note(note_id="n", text="History of Type 2 diabetes mellitus, well controlled.") + clf = NegationTemporalityClassifier() + spans = [_make_span(note, "diabetes")] + out = clf(note, spans) + assert out[0].temporality == Temporality.HISTORICAL + assert is_attributable(out[0]), "Chronic DM should remain attributable even when historical-marked" diff --git a/tests/test_verifier.py b/tests/test_verifier.py new file mode 100644 index 0000000000000000000000000000000000000000..3e4cef9b157260884e0414c98dc7b2a8d8f51892 --- /dev/null +++ b/tests/test_verifier.py @@ -0,0 +1,78 @@ +"""Faithfulness verifier tests — substring check + NLI gating.""" + +from __future__ import annotations + +from app.pipeline.components.i_verifier import ( + NLI_ENTAILMENT_THRESHOLD, + FaithfulnessVerifier, +) +from app.pipeline.types import ( + HCCCandidate, + HierarchyState, + MEATCategory, + MEATEvidence, + Note, +) + + +NOTE_TEXT = "Patient has Type 2 diabetes mellitus. HbA1c 9.2 today. Continued on metformin." + + +def _note() -> Note: + return Note(note_id="n1", text=NOTE_TEXT) + + +def _candidate_with_evidence(evidences: list[MEATEvidence]) -> HCCCandidate: + return HCCCandidate( + hcc="HCC-19", + hcc_label="Diabetes", + raf_weight=0.1, + hierarchy=HierarchyState(group="DIABETES", rank=3), + icd10_links=[], + confidence_raw=0.5, + confidence_calibrated=0.5, + meat_evidence=evidences, + ) + + +def test_substring_match_passes(): + v = FaithfulnessVerifier() + ev = MEATEvidence(category=MEATCategory.EVALUATE, quote="HbA1c 9.2 today.", start=33, end=49) + assert v.verify_substring(_note(), ev) is True + + +def test_fabricated_quote_is_dropped(): + v = FaithfulnessVerifier() + ev = MEATEvidence( + category=MEATCategory.EVALUATE, quote="Patient had an A1c of 6.1.", start=0, end=26 + ) + cand = _candidate_with_evidence([ev]) + verified, drops = v.verify_candidate(_note(), cand) + assert drops == 1 + assert verified.meat_evidence == [] + + +def test_nli_below_threshold_marks_unverified(): + def weak_nli(premise, hypothesis): # noqa: ARG001 + return NLI_ENTAILMENT_THRESHOLD - 0.1 + + v = FaithfulnessVerifier(nli=weak_nli) + ev = MEATEvidence(category=MEATCategory.TREAT, quote="Continued on metformin.", start=51, end=74) + cand = _candidate_with_evidence([ev]) + verified, drops = v.verify_candidate(_note(), cand) + assert drops == 0 + assert len(verified.meat_evidence) == 1 + assert verified.meat_evidence[0].verified is False + assert verified.meat_evidence[0].nli_entailment is not None + + +def test_nli_above_threshold_marks_verified(): + def strong_nli(premise, hypothesis): # noqa: ARG001 + return 0.95 + + v = FaithfulnessVerifier(nli=strong_nli) + ev = MEATEvidence(category=MEATCategory.TREAT, quote="Continued on metformin.", start=51, end=74) + cand = _candidate_with_evidence([ev]) + verified, _drops = v.verify_candidate(_note(), cand) + assert verified.meat_evidence[0].verified is True + assert verified.meat_evidence[0].nli_entailment == 0.95 diff --git a/ui/.eslintrc.cjs b/ui/.eslintrc.cjs new file mode 100644 index 0000000000000000000000000000000000000000..738d9a28290cf7aa71d2a34516e8d1c27ee1990e --- /dev/null +++ b/ui/.eslintrc.cjs @@ -0,0 +1,17 @@ +module.exports = { + root: true, + env: { browser: true, es2022: true }, + extends: [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:react-hooks/recommended", + ], + parser: "@typescript-eslint/parser", + parserOptions: { ecmaVersion: "latest", sourceType: "module" }, + plugins: ["@typescript-eslint", "react-refresh"], + ignorePatterns: ["dist", ".eslintrc.cjs"], + rules: { + "react-refresh/only-export-components": ["warn", { allowConstantExport: true }], + "@typescript-eslint/no-unused-vars": ["warn", { argsIgnorePattern: "^_" }], + }, +}; diff --git a/ui/.prettierrc.json b/ui/.prettierrc.json new file mode 100644 index 0000000000000000000000000000000000000000..bccee911b6b0dac9051d360fbc5e0a6303eed8fb --- /dev/null +++ b/ui/.prettierrc.json @@ -0,0 +1,6 @@ +{ + "semi": true, + "singleQuote": false, + "trailingComma": "all", + "printWidth": 100 +} diff --git a/ui/index.html b/ui/index.html new file mode 100644 index 0000000000000000000000000000000000000000..d0bd00d9d26fe02a6bffaf6dbef27ff875ec17a1 --- /dev/null +++ b/ui/index.html @@ -0,0 +1,13 @@ + + + + + + HCC-MEAT Coder — research demo + + + +
+ + + diff --git a/ui/package.json b/ui/package.json new file mode 100644 index 0000000000000000000000000000000000000000..a4fb7f336bea23c27eb3403f3366d5eecbe8451a --- /dev/null +++ b/ui/package.json @@ -0,0 +1,39 @@ +{ + "name": "hcc-meat-ui", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview", + "lint": "eslint . --ext ts,tsx --max-warnings=0", + "format": "prettier --write ." + }, + "dependencies": { + "@radix-ui/react-tabs": "^1.1.1", + "@radix-ui/react-tooltip": "^1.1.4", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "lucide-react": "^0.460.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "tailwind-merge": "^2.5.4" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@typescript-eslint/eslint-plugin": "^8.15.0", + "@typescript-eslint/parser": "^8.15.0", + "@vitejs/plugin-react": "^4.3.3", + "autoprefixer": "^10.4.20", + "eslint": "^9.15.0", + "eslint-plugin-react-hooks": "^5.0.0", + "eslint-plugin-react-refresh": "^0.4.14", + "postcss": "^8.4.49", + "prettier": "^3.3.3", + "tailwindcss": "^3.4.15", + "typescript": "^5.6.3", + "vite": "^5.4.11" + } +} diff --git a/ui/postcss.config.js b/ui/postcss.config.js new file mode 100644 index 0000000000000000000000000000000000000000..2aa7205d4b402a1bdfbe07110c61df920b370066 --- /dev/null +++ b/ui/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; diff --git a/ui/src/App.tsx b/ui/src/App.tsx new file mode 100644 index 0000000000000000000000000000000000000000..2ae2983c542da9a3470d5027fa734558861c3c38 --- /dev/null +++ b/ui/src/App.tsx @@ -0,0 +1,101 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; +import { DisclaimerBanner } from "./components/DisclaimerBanner"; +import { NotePanel } from "./components/NotePanel"; +import { HCCList } from "./components/HCCList"; +import { DrillIn } from "./components/DrillIn"; +import { codeNote, fetchGallery } from "./lib/api"; +import type { CodingResult, GalleryItem, HCCCandidate } from "./lib/types"; + +const SAMPLE_NOTE = `HPI: 67-year-old male with Type 2 diabetes mellitus with diabetic polyneuropathy. HbA1c 9.2 today, up from 8.4 three months ago. + +PHYSICAL EXAM: Decreased sensation to monofilament bilaterally. + +ASSESSMENT AND PLAN: Type 2 diabetes mellitus with diabetic polyneuropathy, uncontrolled. Continue metformin 1000 mg BID. Start insulin glargine 10 units nightly. Monitor HbA1c in 3 months. Referred to endocrinology.`; + +export default function App() { + const [noteText, setNoteText] = useState(SAMPLE_NOTE); + const [result, setResult] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [selectedHCC, setSelectedHCC] = useState(null); + const [gallery, setGallery] = useState([]); + + useEffect(() => { + fetchGallery() + .then(setGallery) + .catch(() => setGallery([])); + }, []); + + const runPipeline = useCallback(async () => { + setLoading(true); + setError(null); + try { + const r = await codeNote(noteText); + setResult(r); + setSelectedHCC(r.candidates.find((c) => !c.hierarchy.trumped)?.hcc ?? null); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setLoading(false); + } + }, [noteText]); + + const selectedCandidate: HCCCandidate | null = useMemo(() => { + if (!result || !selectedHCC) return null; + return result.candidates.find((c) => c.hcc === selectedHCC) ?? null; + }, [result, selectedHCC]); + + const pickGalleryItem = useCallback((item: GalleryItem) => { + setNoteText(item.text); + setResult(item.result); + setSelectedHCC(item.result.candidates.find((c) => !c.hierarchy.trumped)?.hcc ?? null); + }, []); + + return ( +
+ + +
+
+

HCC-MEAT Coder

+ + CMS-HCC v28 · MEAT-justified · calibrated · RADV-framed + +
+
+ +
+
+ +
+ +
+ +
+ +
+ +
+
+ +
+ Research demo · synthetic data only · not for clinical or billing use ·{" "} + {result?.latency_ms ? `${Math.round(result.latency_ms)}ms` : "—"} +
+
+ ); +} diff --git a/ui/src/components/DisclaimerBanner.tsx b/ui/src/components/DisclaimerBanner.tsx new file mode 100644 index 0000000000000000000000000000000000000000..44e7db40214e5d749773c4d8556e12a86627449f --- /dev/null +++ b/ui/src/components/DisclaimerBanner.tsx @@ -0,0 +1,9 @@ +export function DisclaimerBanner() { + return ( +
+ Research demo. Not for clinical, billing, or coding use. Input notes are + processed in-memory and are not stored or logged. Only synthetic or publicly-licensed + case data is used in the preloaded gallery. +
+ ); +} diff --git a/ui/src/components/DrillIn.tsx b/ui/src/components/DrillIn.tsx new file mode 100644 index 0000000000000000000000000000000000000000..bf026d33b9be8cd5c04316bd50b974b2bfe8cf62 --- /dev/null +++ b/ui/src/components/DrillIn.tsx @@ -0,0 +1,132 @@ +import type { HCCCandidate, MEATCat, MEATEvidence } from "../lib/types"; +import { cn } from "../lib/cn"; + +const CATEGORY_LABEL: Record = { + M: "Monitor", + E: "Evaluate", + A: "Assess", + T: "Treat", +}; + +const CATEGORY_HINT: Record = { + M: "Signs / symptoms actively tracked", + E: "Tests / exams reviewed", + A: "Clinician impression documented", + T: "Treatment prescribed, continued, or adjusted", +}; + +export function DrillIn({ candidate }: { candidate: HCCCandidate | null }) { + if (!candidate) { + return ( +
+ Select an HCC to inspect its MEAT evidence and risk breakdown. +
+ ); + } + + const byCategory: Record = { M: [], E: [], A: [], T: [] }; + for (const ev of candidate.meat_evidence) byCategory[ev.category].push(ev); + + return ( +
+
+
+

{candidate.hcc}

+ RAF {candidate.raf_weight.toFixed(3)} +
+
{candidate.hcc_label}
+
+ +
+
+

+ MEAT evidence +

+
    + {(["M", "E", "A", "T"] as MEATCat[]).map((cat) => ( + + ))} +
+
+ +
+

+ Over-coding risk +

+
+
+ + {candidate.risk?.level.toUpperCase() ?? "—"} + + + score {candidate.risk?.score.toFixed(3) ?? "—"} + +
+
    + {(candidate.risk?.reasons ?? []).map((r, i) => ( +
  • {r}
    • + ))} +
+
+
+ +
+

+ Supporting ICD-10 codes +

+
    + {candidate.icd10_links.map((lnk) => ( +
  • + {lnk.icd10} + {lnk.description} + linker {lnk.linker_confidence.toFixed(2)} +
    • + ))} +
+
+
+
+ ); +} + +function MEATRow({ cat, evidences }: { cat: MEATCat; evidences: MEATEvidence[] }) { + const verifiedCount = evidences.filter((e) => e.verified).length; + const hasEvidence = verifiedCount > 0; + return ( +
  • +
    + + {cat} — {CATEGORY_LABEL[cat]} + + + {hasEvidence ? `${verifiedCount} verified` : "Insufficient"} + +
    +

    {CATEGORY_HINT[cat]}

    + {evidences.length === 0 ? ( +

    No quote supporting this category.

    + ) : ( +
      + {evidences.map((ev, i) => ( +
    • + "{ev.quote}" +
      + {ev.section_kind ?? "—"} · extractor {ev.extractor_confidence.toFixed(2)} + {ev.nli_entailment != null && ` · nli ${ev.nli_entailment.toFixed(2)}`} + {!ev.verified && " · unverified"} +
      +
      • + ))} +
    + )} +
    • + ); +} diff --git a/ui/src/components/HCCList.tsx b/ui/src/components/HCCList.tsx new file mode 100644 index 0000000000000000000000000000000000000000..2f20ec73f8d0297399cd108a96eaab4df15679ad --- /dev/null +++ b/ui/src/components/HCCList.tsx @@ -0,0 +1,94 @@ +import type { CodingResult, HCCCandidate } from "../lib/types"; +import { cn } from "../lib/cn"; + +interface Props { + result: CodingResult | null; + error: string | null; + selectedHCC: string | null; + onSelect: (hcc: string) => void; +} + +export function HCCList({ result, error, selectedHCC, onSelect }: Props) { + return ( +
    +
    +

    Suggested capture

    + + {result ? `${result.candidates.filter((c) => !c.hierarchy.trumped).length} active` : ""} + +
    + + {error && ( +
    + {error} +
    + )} + +
      + {(result?.candidates ?? []).map((c) => ( + onSelect(c.hcc)} + /> + ))} + {!result &&
    • Click "Code note" to run.
      • } +
    +
    + ); +} + +function HCCRow({ + candidate, + selected, + onClick, +}: { + candidate: HCCCandidate; + selected: boolean; + onClick: () => void; +}) { + const cov = meatCoverageSymbols(candidate); + const risk = candidate.risk; + + return ( +
  • +
    + {candidate.hcc} + + {(risk?.level ?? "?").toUpperCase()} + +
    +
    {candidate.hcc_label}
    +
    + RAF {candidate.raf_weight.toFixed(3)} + Conf {candidate.confidence_calibrated.toFixed(2)} + {cov} + {candidate.hierarchy.trumped && ( + + trumped by {candidate.hierarchy.trumped_by} + + )} +
    +
    • + ); +} + +function meatCoverageSymbols(c: HCCCandidate): string { + const filled = new Set(c.meat_evidence.filter((e) => e.verified).map((e) => e.category)); + return (["M", "E", "A", "T"] as const).map((cat) => (filled.has(cat) ? cat : "·")).join(""); +} + +function riskColor(level: string | undefined) { + if (level === "low") return "text-risk-low"; + if (level === "medium") return "text-risk-medium"; + if (level === "high") return "text-risk-high"; + return "text-slate-400"; +} diff --git a/ui/src/components/NotePanel.tsx b/ui/src/components/NotePanel.tsx new file mode 100644 index 0000000000000000000000000000000000000000..9837dcb354a41a7b71532fb10ae3e0aea04025a1 --- /dev/null +++ b/ui/src/components/NotePanel.tsx @@ -0,0 +1,120 @@ +import { useMemo } from "react"; +import type { CodingResult, GalleryItem, HCCCandidate, MEATEvidence } from "../lib/types"; +import { cn } from "../lib/cn"; + +interface Props { + noteText: string; + onChange: (text: string) => void; + gallery: GalleryItem[]; + onPickGalleryItem: (item: GalleryItem) => void; + onCode: () => void; + loading: boolean; + result: CodingResult | null; + selectedCandidate: HCCCandidate | null; +} + +export function NotePanel(props: Props) { + const { noteText, onChange, gallery, onPickGalleryItem, onCode, loading, selectedCandidate } = + props; + + const highlights = useMemo(() => buildHighlights(selectedCandidate, noteText), [ + selectedCandidate, + noteText, + ]); + + return ( +
    +
    +

    Clinical note

    +
    + + +
    +
    + +
    +