Spaces:
Sleeping
Sleeping
File size: 48,782 Bytes
3863567 1da8d42 3863567 1da8d42 8667e08 3863567 1da8d42 3863567 1da8d42 8667e08 27697e4 1da8d42 8667e08 27697e4 8667e08 3863567 8667e08 3863567 8667e08 3863567 27697e4 8667e08 3863567 1da8d42 3863567 1da8d42 3863567 1da8d42 3863567 1da8d42 3863567 1da8d42 3863567 1da8d42 3863567 1da8d42 3863567 27697e4 3863567 27697e4 3863567 27697e4 3863567 27697e4 1da8d42 3863567 27697e4 3863567 8667e08 3863567 8667e08 3863567 8667e08 27697e4 3863567 8667e08 3863567 8667e08 3863567 8667e08 3863567 8667e08 27697e4 3863567 8667e08 27697e4 8667e08 3863567 8667e08 3863567 8667e08 3863567 1da8d42 3863567 1da8d42 3863567 27697e4 3863567 8667e08 3863567 8667e08 3863567 8667e08 3863567 8667e08 3863567 8667e08 3863567 1da8d42 3863567 8667e08 3863567 1da8d42 27697e4 1da8d42 3863567 1da8d42 8667e08 27697e4 1da8d42 3863567 27697e4 3863567 1da8d42 8667e08 27697e4 1da8d42 3863567 1da8d42 3863567 1da8d42 3863567 27697e4 3863567 8667e08 1da8d42 27697e4 3863567 1da8d42 3863567 1da8d42 27697e4 3863567 27697e4 3863567 27697e4 3863567 27697e4 3863567 27697e4 3863567 27697e4 3863567 27697e4 3863567 27697e4 3863567 8667e08 3863567 8667e08 3863567 27697e4 3863567 27697e4 3863567 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 | """
Medical Coding Auditor Environment — Core Logic.
The agent plays the role of a Medical Coding Auditor performing a pre-bill review.
It validates a proposed set of ICD-10 and CPT codes against a clinical note and
patient demographics without assigning new codes.
Reward shaping (dense, multi-signal):
+0.10 QueryGuideline on a code present in the proposed set
+0.30 FlagError that correctly identifies an NCCI edit or Excludes1 conflict
+0.20 FlagError that correctly identifies a demographic mismatch,
specificity error, or untraceable code
+0.05 Justification quality bonus (key terms from expected error description)
+0.05 Investigation coherence bonus (queried code's guideline before flagging it)
-0.05 Flag without investigation penalty (flagging a code without querying first)
-0.10 Repeated QueryGuideline / CheckNCCIEdits on an already-queried code/pair
-0.10 FlagError with correct code but wrong error_type classification
-0.20 FlagError on a code with no expected error (false positive)
-0.50 Any action that references a code NOT in the proposed set (hallucination)
+final Terminal reward on SubmitAudit:
grader_score - 0.15 * false_positives + 0.1 * (max_steps - steps) / max_steps
+ 0.05 thorough_review bonus (queried all proposed codes before submitting)
+0.15 Relevant clarifying question (matches trigger keywords in clarification pool)
-0.10 Irrelevant or repeated clarifying question
+0.10 Terminal bonus for asking necessary clarifications before flagging
Auto-termination: episode ends with auto-submit and -0.1 penalty at max_steps
Hierarchical reward (HERON-style):
Wrong error_type penalty scaled by conceptual distance between error types.
Close misclassifications (e.g. excludes1↔ncci_edit) penalized less than
distant ones (e.g. demographic_mismatch↔untraceable_code).
Partial observability mode:
When enabled via reset(partial_observability=True), the clinical note is
progressively revealed as the agent queries codes. Initially only the
REASON FOR VISIT section is visible. Each query_guideline reveals the
next section. This models real-world information gathering.
"""
from __future__ import annotations
import json
import os
import re
from pathlib import Path
from typing import Any, Dict, List, Optional, Set
from uuid import uuid4
from openenv.core.env_server.interfaces import Environment
from openenv.core.env_server.types import State
try:
from ..models import MedicalCodingAction, MedicalCodingObservation
except ImportError:
try:
from models import MedicalCodingAction, MedicalCodingObservation
except ImportError:
from medical_coding_env.models import MedicalCodingAction, MedicalCodingObservation
# ---------------------------------------------------------------------------
# Locate the ground truth data file
# ---------------------------------------------------------------------------
_DATA_FILE = Path(__file__).parent.parent / "data" / "ground_truth_cases.json"
def _load_data() -> Dict[str, Any]:
with open(_DATA_FILE, "r", encoding="utf-8") as fh:
return json.load(fh)
# ---------------------------------------------------------------------------
# Feature I: HERON-style error_type distance matrix
# ---------------------------------------------------------------------------
# Distance 0 = same type, 1 = close (same category), 2 = far (different category)
# Categories: "conflict" (excludes1, ncci_edit), "validity" (specificity, untraceable),
# "demographic" (demographic_mismatch)
_ERROR_TYPE_DISTANCE: Dict[tuple[str, str], int] = {}
_ERROR_TYPES = [
"demographic_mismatch", "excludes1_conflict", "ncci_edit",
"specificity_error", "untraceable_code",
]
_ERROR_CATEGORIES = {
"excludes1_conflict": "conflict",
"ncci_edit": "conflict",
"specificity_error": "validity",
"untraceable_code": "validity",
"demographic_mismatch": "demographic",
}
for _a in _ERROR_TYPES:
for _b in _ERROR_TYPES:
if _a == _b:
_ERROR_TYPE_DISTANCE[(_a, _b)] = 0
elif _ERROR_CATEGORIES[_a] == _ERROR_CATEGORIES[_b]:
_ERROR_TYPE_DISTANCE[(_a, _b)] = 1
else:
_ERROR_TYPE_DISTANCE[(_a, _b)] = 2
# ---------------------------------------------------------------------------
# Feature L: Frequency-weighted asymmetric reward multipliers
# ---------------------------------------------------------------------------
# Rarity reflects how often each error type occurs in real-world billing audits.
# Rare, subtle errors are disproportionately rewarded (ASL-inspired).
_ERROR_RARITY: Dict[str, float] = {
"demographic_mismatch": 1.0, # common, visually obvious
"ncci_edit": 1.2, # moderate — requires NCCI lookup
"excludes1_conflict": 1.2, # moderate — requires guideline knowledge
"untraceable_code": 1.3, # less common — requires code validation
"specificity_error": 1.5, # rare, subtle — 7th character nuance
}
# ---------------------------------------------------------------------------
# Feature K: Adaptive curriculum learning — class-level state
# ---------------------------------------------------------------------------
_CURRICULUM_LEVELS = ["easy", "medium", "hard", "expert"]
_CURRICULUM_PROMOTE_THRESHOLD = 0.8 # avg score to move up
_CURRICULUM_DEMOTE_THRESHOLD = 0.3 # avg score to move down
_CURRICULUM_WINDOW = 5 # episodes to average over
# ---------------------------------------------------------------------------
# Partial Observability — Clinical Note Section Parser
# ---------------------------------------------------------------------------
# Common clinical note section headers
_SECTION_PATTERN = re.compile(
r"^(REASON FOR VISIT|HISTORY OF PRESENT ILLNESS|PAST MEDICAL HISTORY|"
r"SOCIAL HISTORY|REVIEW OF SYSTEMS|MEDICATIONS|PHYSICAL EXAM|"
r"EXAMINATION|IMAGING|SPIROMETRY[^:]*|SERVICES RENDERED|"
r"ENDOCRINOLOGY ASSESSMENT|CARDIOLOGY ASSESSMENT|RIGHT ARM|"
r"ASSESSMENT AND PLAN|ASSESSMENT|PLAN)\s*:",
re.MULTILINE | re.IGNORECASE,
)
def _parse_note_sections(clinical_note: str) -> List[tuple[str, str]]:
"""
Parse a clinical note into (header, body) sections.
Returns at least one section. Unparseable notes return as a single section.
"""
matches = list(_SECTION_PATTERN.finditer(clinical_note))
if not matches:
return [("CLINICAL NOTE", clinical_note)]
sections: List[tuple[str, str]] = []
for i, match in enumerate(matches):
header = match.group(1).upper()
start = match.end()
end = matches[i + 1].start() if i + 1 < len(matches) else len(clinical_note)
body = clinical_note[start:end].strip()
sections.append((header, body))
return sections
def _build_visible_note(
sections: List[tuple[str, str]],
revealed_count: int,
) -> str:
"""Build the currently visible clinical note from revealed sections."""
if not sections:
return ""
visible = sections[:revealed_count]
hidden_count = len(sections) - revealed_count
parts = [f"{header}: {body}" for header, body in visible]
if hidden_count > 0:
parts.append(f"\n[{hidden_count} more section(s) not yet revealed — query codes to investigate further]")
return "\n\n".join(parts)
# ---------------------------------------------------------------------------
# Deterministic Grader
# ---------------------------------------------------------------------------
def _grade_audit(
draft_report: List[Dict[str, Any]],
expected_errors: List[Dict[str, Any]],
) -> tuple[float, int]:
"""
Compare the agent's submitted audit report against the ground truth.
Scoring logic:
- For each expected error, check if the agent correctly flagged it.
- A flag is "correct" if both the code AND error_type match → 1.0 credit.
- Partial credit (0.5x) if the correct code is flagged but with a wrong error_type.
- False positives (flags on codes with no expected error) are counted and penalized
in the terminal reward calculation.
- Final base score = sum(credit for each expected error) / len(expected_errors)
Returns (base_score, false_positive_count).
"""
if not expected_errors:
return 1.0, 0
expected_codes = {e["code"] for e in expected_errors}
flagged_by_code: Dict[str, List[str]] = {}
false_positives = 0
for entry in draft_report:
code = entry.get("code", "")
if code not in flagged_by_code:
flagged_by_code[code] = []
flagged_by_code[code].append(entry.get("error_type", ""))
if code not in expected_codes:
false_positives += 1
total_credit = 0.0
for expected in expected_errors:
exp_code = expected["code"]
exp_error_type = expected["error_type"]
if exp_code not in flagged_by_code:
continue
agent_error_types = flagged_by_code[exp_code]
# Feature L: rarity multiplier applied to grader credit
rarity = _ERROR_RARITY.get(exp_error_type, 1.0)
if exp_error_type in agent_error_types:
total_credit += 1.0 * rarity
else:
# HERON: partial credit scaled by conceptual distance of best agent guess
best_distance = min(
_ERROR_TYPE_DISTANCE.get((atype, exp_error_type), 2)
for atype in agent_error_types
)
# distance=1 (close miss) → 0.55 credit, distance=2 (far miss) → 0.4 credit
partial = max(0.4, 0.7 - 0.15 * best_distance)
total_credit += partial * rarity
# Normalize by total possible credit (sum of rarity weights, not raw count)
max_credit = sum(_ERROR_RARITY.get(e["error_type"], 1.0) for e in expected_errors)
base_score = round(total_credit / max(max_credit, 1.0), 4)
return base_score, false_positives
# ---------------------------------------------------------------------------
# Environment
# ---------------------------------------------------------------------------
class MedicalCodingEnvironment(Environment):
"""
Medical Coding Auditor OpenEnv Environment.
The agent reviews a case containing patient demographics, a clinical note,
and a proposed set of ICD-10-CM / CPT billing codes. It uses four tools:
- query_guideline: fetch coding rules for a specific code
- check_ncci_edits: check for CMS NCCI PTP bundling conflicts
- flag_error: add an entry to the draft audit report
- submit_audit: end the episode and trigger the grader
Task selection: pass task_id as a keyword argument to reset().
Available task_ids: easy_demographic, medium_ncci_conflict, hard_specificity_untraceable
The environment supports concurrent WebSocket sessions.
"""
SUPPORTS_CONCURRENT_SESSIONS: bool = True
# Feature K: class-level curriculum state (shared across episodes)
_performance_history: Dict[str, List[float]] = {
level: [] for level in _CURRICULUM_LEVELS
}
_curriculum_level: str = "easy"
# Default task cycling order for sequential inference runs
_TASK_ORDER = [
"easy_demographic",
"medium_ncci_conflict",
"medium_excludes1",
"hard_specificity_untraceable",
"expert_multi_error",
]
def __init__(self) -> None:
self._data = _load_data()
self._state = State(episode_id=str(uuid4()), step_count=0)
self._task: Optional[Dict[str, Any]] = None
self._proposed_codes: Dict[str, Dict[str, str]] = {}
self._draft_report: List[Dict[str, Any]] = []
self._codes_queried: List[str] = []
self._pairs_checked: List[str] = []
self._done: bool = False
self._grader_score: Optional[float] = None
self._max_steps: int = 20
# Feature F: investigation strategy tracking
self._action_history: List[tuple[str, Optional[str]]] = []
# Feature E: partial observability
self._partial_observability: bool = False
self._note_sections: List[tuple[str, str]] = [] # (header, body) pairs
self._revealed_section_count: int = 0
# Feature G: physician clarifications
self._clarifications_asked: List[str] = []
# Feature H: evidence extraction
self._extracted_evidence: List[str] = [] # all extracted spans
self._evidence_covered_codes: Set[str] = set() # codes with relevant evidence
# Feature M: episode metrics (populated at episode end)
self._episode_metrics: Optional[Dict[str, Any]] = None
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def reset(self, task_id: Optional[str] = None, partial_observability: Optional[bool] = None, **kwargs: Any) -> MedicalCodingObservation: # type: ignore[override]
"""
Reset the environment for a new episode.
Args:
task_id: Which task to load. If None, cycles through tasks in order.
Use 'random' for a procedurally generated case.
partial_observability: If True, the clinical note is progressively
revealed as the agent queries codes. Default: False.
Returns:
Initial MedicalCodingObservation with the case context.
"""
self._state = State(episode_id=str(uuid4()), step_count=0)
self._draft_report = []
self._codes_queried = []
self._pairs_checked = []
self._done = False
self._grader_score = None
self._action_history = []
self._clarifications_asked = []
self._extracted_evidence = []
self._evidence_covered_codes = set()
self._episode_metrics = None
# Select task
tasks = {t["task_id"]: t for t in self._data["tasks"]}
if task_id and task_id.startswith("random"):
from data.case_generator import generate_random_task
if "_" in task_id:
# Explicit difficulty suffix: random_easy, random_medium, etc.
difficulty = task_id.split("_", 1)[1]
else:
# Feature K: no suffix → use adaptive curriculum level
difficulty = self._update_curriculum()
self._task = generate_random_task(
self._data, self._state.episode_id, difficulty=difficulty,
)
elif task_id and task_id in tasks:
self._task = tasks[task_id]
else:
idx = hash(self._state.episode_id) % len(self._TASK_ORDER)
self._task = tasks[self._TASK_ORDER[idx]]
self._proposed_codes = self._task["proposed_codes"]
self._max_steps = self._task["max_steps"]
# Partial observability setup
self._partial_observability = bool(partial_observability)
if self._partial_observability:
self._note_sections = _parse_note_sections(self._task["clinical_note"])
self._revealed_section_count = 1 # start with just REASON FOR VISIT
else:
self._note_sections = []
self._revealed_section_count = 0
po_note = " [PARTIAL OBSERVABILITY: clinical note will be revealed as you investigate.]" if self._partial_observability else ""
return self._build_observation(
tool_result=(
f"Audit session started. Task: {self._task['task_id']} "
f"({self._task['difficulty']} difficulty). "
f"Review the patient demographics, clinical note, and proposed codes. "
f"Use query_guideline, check_ncci_edits, flag_error, then submit_audit."
f"{po_note}"
),
reward=0.0,
done=False,
)
def step(self, action: MedicalCodingAction) -> MedicalCodingObservation: # type: ignore[override]
"""
Execute one auditor action and return the resulting observation.
Dense reward shaping is applied per the specification.
Auto-terminates at max_steps with a timeout penalty.
"""
if self._done:
return self._build_observation(
tool_result="Episode already ended. Call reset() to start a new audit.",
reward=0.0,
done=True,
error="Episode is done. Call reset().",
)
self._state.step_count += 1
# Auto-termination: if this is the last allowed step, force submit
if self._state.step_count >= self._max_steps and action.action_type != "submit_audit":
return self._handle_auto_termination()
# Route to the appropriate handler
handler = {
"query_guideline": self._handle_query_guideline,
"check_ncci_edits": self._handle_check_ncci_edits,
"flag_error": self._handle_flag_error,
"ask_clarifying_question": self._handle_ask_clarification,
"extract_evidence": self._handle_extract_evidence,
"submit_audit": self._handle_submit_audit,
}.get(action.action_type)
if handler is None:
return self._build_observation(
tool_result=f"Unknown action_type: {action.action_type}",
reward=-0.2,
done=False,
error=f"Unknown action_type: {action.action_type}",
)
return handler(action)
@property
def state(self) -> State:
return self._state
# ------------------------------------------------------------------
# Action handlers
# ------------------------------------------------------------------
def _handle_query_guideline(self, action: MedicalCodingAction) -> MedicalCodingObservation:
"""Look up coding guidelines for a specific code."""
code = action.code
if not code:
self._action_history.append(("query_guideline", None))
return self._build_observation(
tool_result="query_guideline requires a 'code' field.",
reward=-0.2,
done=False,
error="Missing required field: code",
)
self._action_history.append(("query_guideline", code))
# Hallucination check — code must be in the proposed set
if code not in self._proposed_codes:
return self._build_observation(
tool_result=(
f"Code '{code}' is not in the proposed code set for this encounter. "
f"Proposed codes are: {list(self._proposed_codes.keys())}. "
f"Only query codes that appear in the proposed billing set."
),
reward=-0.5,
done=False,
error=f"Hallucinated code: '{code}' is not in the proposed code set.",
)
# Repeated query penalty
if code in self._codes_queried:
return self._build_observation(
tool_result=f"You have already queried guidelines for '{code}' this session. Check your draft report.",
reward=-0.1,
done=False,
error=f"Repeated query: '{code}' already queried.",
)
# Look up guideline from data
guideline = self._lookup_guideline(code)
self._codes_queried.append(code)
# Partial observability: reveal next section
if self._partial_observability and self._revealed_section_count < len(self._note_sections):
self._revealed_section_count += 1
return self._build_observation(
tool_result=guideline,
reward=0.1,
done=False,
)
def _handle_check_ncci_edits(self, action: MedicalCodingAction) -> MedicalCodingObservation:
"""Check CMS NCCI PTP edit table for two codes."""
code1, code2 = action.code1, action.code2
self._action_history.append(("check_ncci_edits", f"{code1}|{code2}"))
if not code1 or not code2:
return self._build_observation(
tool_result="check_ncci_edits requires both 'code1' and 'code2' fields.",
reward=-0.2,
done=False,
error="Missing required field(s): code1 and/or code2",
)
# Hallucination check — both codes must be in the proposed set
hallucinated = [c for c in (code1, code2) if c not in self._proposed_codes]
if hallucinated:
return self._build_observation(
tool_result=(
f"Code(s) {hallucinated} are not in the proposed code set. "
f"Only check NCCI edits for codes in the proposed billing set: "
f"{list(self._proposed_codes.keys())}."
),
reward=-0.5,
done=False,
error=f"Hallucinated code(s): {hallucinated}",
)
# Repeated check penalty
pair_key = "|".join(sorted([code1, code2]))
if pair_key in self._pairs_checked:
return self._build_observation(
tool_result=f"You have already checked the NCCI edit between '{code1}' and '{code2}'.",
reward=-0.1,
done=False,
error=f"Repeated NCCI check: {pair_key}",
)
# Look up NCCI edit
ncci_result, found_edit = self._lookup_ncci_edit(code1, code2)
self._pairs_checked.append(pair_key)
# Small positive reward for checking a valid pair (no hallucination)
reward = 0.05 if not found_edit else 0.1
return self._build_observation(
tool_result=ncci_result,
reward=reward,
done=False,
)
def _handle_flag_error(self, action: MedicalCodingAction) -> MedicalCodingObservation:
"""Record a coding error in the draft audit report."""
code = action.code
error_type = action.error_type
justification = action.justification
self._action_history.append(("flag_error", code))
if not code or not error_type or not justification:
missing = [f for f, v in [("code", code), ("error_type", error_type), ("justification", justification)] if not v]
return self._build_observation(
tool_result=f"flag_error requires: code, error_type, justification. Missing: {missing}",
reward=-0.2,
done=False,
error=f"Missing required field(s): {missing}",
)
# Hallucination check — code must be in the proposed set
if code not in self._proposed_codes:
return self._build_observation(
tool_result=(
f"Cannot flag '{code}' — it is not in the proposed code set. "
f"Only flag codes that appear in the proposed billing set: "
f"{list(self._proposed_codes.keys())}."
),
reward=-0.5,
done=False,
error=f"Hallucinated code: '{code}' is not in the proposed code set.",
)
# Repeated flag check
already_flagged = any(
e["code"] == code and e["error_type"] == error_type
for e in self._draft_report
)
if already_flagged:
return self._build_observation(
tool_result=f"Code '{code}' has already been flagged as '{error_type}'. Avoid duplicate flags.",
reward=-0.1,
done=False,
error=f"Duplicate flag: '{code}' / '{error_type}'",
)
# Evaluate correctness against ground truth
expected_errors = {e["code"]: e["error_type"] for e in self._task["expected_errors"]}
reward, feedback = self._score_flag(code, error_type, justification, expected_errors)
# Feature F: investigation coherence bonus/penalty
investigated = self._was_investigated(code, error_type)
if investigated:
reward += 0.05
feedback += " (Investigation coherence bonus: +0.05)"
else:
reward -= 0.05
feedback += " (No prior investigation of this code: -0.05)"
# Record in draft report
self._draft_report.append({
"code": code,
"error_type": error_type,
"justification": justification,
"step": self._state.step_count,
})
return self._build_observation(
tool_result=f"Error flagged for '{code}' as '{error_type}'. {feedback}",
reward=reward,
done=False,
)
def _handle_submit_audit(self, action: MedicalCodingAction) -> MedicalCodingObservation:
"""End the episode and grade the submitted audit report."""
return self._finalize_episode(timeout=False)
def _handle_auto_termination(self) -> MedicalCodingObservation:
"""Auto-submit when max_steps is reached. Applies a timeout penalty."""
return self._finalize_episode(timeout=True)
def _handle_ask_clarification(self, action: MedicalCodingAction) -> MedicalCodingObservation:
"""
Simulate asking the physician a clarifying question.
The environment matches the question against keyword triggers in the task's
clarification_pool. A relevant question yields a physician response and a
+0.15 reward. Irrelevant or repeated questions are penalized.
"""
question = action.question
self._action_history.append(("ask_clarifying_question", question))
if not question:
return self._build_observation(
tool_result="ask_clarifying_question requires a 'question' field.",
reward=-0.2,
done=False,
error="Missing required field: question",
)
# Repeated question penalty
if question in self._clarifications_asked:
return self._build_observation(
tool_result="You have already asked this question. Avoid repeating clarifications.",
reward=-0.1,
done=False,
error="Repeated clarifying question.",
)
self._clarifications_asked.append(question)
# Match against clarification_pool (keyword-based)
pool = self._task.get("clarification_pool", [])
question_lower = question.lower()
for entry in pool:
triggers = entry.get("triggers", [])
if any(t.lower() in question_lower for t in triggers):
response = entry["response"]
return self._build_observation(
tool_result=(
f"PHYSICIAN RESPONSE: {response}"
),
reward=0.15,
done=False,
)
# No matching clarification — irrelevant question
return self._build_observation(
tool_result=(
"PHYSICIAN RESPONSE: I don't have additional information on that. "
"Please proceed with the available documentation."
),
reward=-0.1,
done=False,
error="Irrelevant clarifying question (no matching trigger in task pool).",
)
def _handle_extract_evidence(self, action: MedicalCodingAction) -> MedicalCodingObservation:
"""
Feature H: Extract a text span from the clinical note as evidence.
Validates the span exists verbatim in the note (hallucination guard).
Checks if it overlaps with expected evidence spans — rewards grounded reasoning.
Tracks which codes have documented evidence for the flag_error bonus.
"""
span = action.evidence_text
self._action_history.append(("extract_evidence", span))
if not span:
return self._build_observation(
tool_result="extract_evidence requires an 'evidence_text' field.",
reward=-0.2,
done=False,
error="Missing required field: evidence_text",
)
clinical_note = self._task["clinical_note"]
# Hallucination guard: span must appear verbatim in the note
if span not in clinical_note:
return self._build_observation(
tool_result=(
f"Evidence text not found verbatim in the clinical note. "
f"Only extract exact substrings from the note."
),
reward=-0.05,
done=False,
error="Hallucinated evidence: span not found in clinical note.",
)
# Repeated extraction penalty
if span in self._extracted_evidence:
return self._build_observation(
tool_result="This evidence span has already been extracted.",
reward=-0.05,
done=False,
error="Repeated evidence extraction.",
)
self._extracted_evidence.append(span)
span_lower = span.lower()
# Check against expected error evidence spans
matched_codes = []
for exp in self._task.get("expected_errors", []):
for evidence_span in exp.get("evidence_spans", []):
if evidence_span.lower() in span_lower or span_lower in evidence_span.lower():
matched_codes.append(exp["code"])
self._evidence_covered_codes.add(exp["code"])
break
if matched_codes:
return self._build_observation(
tool_result=(
f"Evidence extracted: \"{span[:120]}\"\n"
f"This span is relevant to coding decision(s) for: {matched_codes}."
),
reward=0.05,
done=False,
)
# Valid span but not linked to any expected error
return self._build_observation(
tool_result=(
f"Evidence extracted: \"{span[:120]}\"\n"
f"This span does not appear to be directly relevant to the expected coding errors. "
f"Continue investigating."
),
reward=0.0,
done=False,
)
def _finalize_episode(self, timeout: bool) -> MedicalCodingObservation:
"""
Grade the audit and compute the terminal reward.
Terminal reward = base_grader_score
- 0.15 * false_positive_count
+ efficiency_bonus (0.1 * remaining_steps / max_steps)
- timeout_penalty (0.1 if auto-terminated)
Clamped to [0.0, 1.0].
"""
expected_errors = self._task["expected_errors"]
base_score, fp_count = _grade_audit(self._draft_report, expected_errors)
# Efficiency bonus: reward finishing early
steps_used = self._state.step_count
efficiency_bonus = 0.1 * max(self._max_steps - steps_used, 0) / self._max_steps
# False positive penalty
fp_penalty = 0.15 * fp_count
# Timeout penalty
timeout_penalty = 0.1 if timeout else 0.0
# Thorough review bonus
review_bonus = self._thorough_review_bonus() if not timeout else 0.0
# Feature G: clarification terminal bonus
# +0.10 if agent asked at least one relevant clarification before flagging
clarification_bonus = 0.0
pool = self._task.get("clarification_pool", [])
if pool and self._clarifications_asked:
# Check if any asked question matched a pool trigger
matched = False
for entry in pool:
triggers = entry.get("triggers", [])
for q in self._clarifications_asked:
if any(t.lower() in q.lower() for t in triggers):
matched = True
break
if matched:
break
if matched:
clarification_bonus = 0.10
terminal_reward = (
base_score
- fp_penalty
+ efficiency_bonus
- timeout_penalty
+ review_bonus
+ clarification_bonus
)
terminal_reward = round(max(min(terminal_reward, 1.0), 0.0), 4)
self._grader_score = terminal_reward
self._done = True
# Feature K: record score in curriculum history
difficulty = self._task.get("difficulty", "easy")
if difficulty in MedicalCodingEnvironment._performance_history:
MedicalCodingEnvironment._performance_history[difficulty].append(terminal_reward)
# Feature M: compute episode metrics
self._episode_metrics = self._compute_episode_metrics(base_score, fp_count)
errors_found = len(self._draft_report)
errors_expected = len(expected_errors)
summary_lines = [
f"AUDIT {'AUTO-SUBMITTED (max steps reached)' if timeout else 'SUBMITTED'}.",
f"Base grader score: {base_score:.2f} | Efficiency bonus: +{efficiency_bonus:.2f}"
+ (f" | Thorough review bonus: +{review_bonus:.2f}" if review_bonus > 0 else "")
+ (f" | Clarification bonus: +{clarification_bonus:.2f}" if clarification_bonus > 0 else "")
+ (f" | Timeout penalty: -{timeout_penalty:.2f}" if timeout else "")
+ (f" | False positive penalty: -{fp_penalty:.2f}" if fp_count > 0 else ""),
f"Final score: {terminal_reward:.2f}",
f"Errors flagged: {errors_found} | Errors expected: {errors_expected}",
]
for exp in expected_errors:
flagged = any(
e["code"] == exp["code"] and e["error_type"] == exp["error_type"]
for e in self._draft_report
)
status = "FOUND" if flagged else "MISSED"
summary_lines.append(f" [{status}] {exp['code']} ({exp['error_type']})")
if fp_count > 0:
expected_codes = {e["code"] for e in expected_errors}
fp_entries = [e["code"] for e in self._draft_report if e["code"] not in expected_codes]
summary_lines.append(f" False positives: {fp_entries}")
return self._build_observation(
tool_result="\n".join(summary_lines),
reward=terminal_reward,
done=True,
)
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def _update_curriculum(self) -> str:
"""
Feature K: Adaptive curriculum learning (HiCu-style).
Reads the class-level performance history to decide what difficulty to use
for the next random episode. Promotes when avg score > 0.8 over the last
N episodes; demotes when avg < 0.3. Returns the chosen difficulty string.
"""
level = MedicalCodingEnvironment._curriculum_level
history = MedicalCodingEnvironment._performance_history.get(level, [])
if len(history) >= _CURRICULUM_WINDOW:
avg = sum(history[-_CURRICULUM_WINDOW:]) / _CURRICULUM_WINDOW
idx = _CURRICULUM_LEVELS.index(level)
if avg >= _CURRICULUM_PROMOTE_THRESHOLD and idx < len(_CURRICULUM_LEVELS) - 1:
MedicalCodingEnvironment._curriculum_level = _CURRICULUM_LEVELS[idx + 1]
elif avg <= _CURRICULUM_DEMOTE_THRESHOLD and idx > 0:
MedicalCodingEnvironment._curriculum_level = _CURRICULUM_LEVELS[idx - 1]
return MedicalCodingEnvironment._curriculum_level
def _lookup_guideline(self, code: str) -> str:
"""Return official guideline text for a given code."""
icd10 = self._data.get("icd10_guidelines", {})
cpt = self._data.get("cpt_guidelines", {})
if code in icd10:
g = icd10[code]
parts = [
f"ICD-10-CM Guideline for {code}: {g['description']}",
f"Official Note: {g['note']}",
]
if g.get("gender_restriction"):
parts.append(f"Gender Restriction: APPLICABLE TO {g['gender_restriction'].upper()} PATIENTS ONLY.")
if g.get("seventh_char_guidance"):
parts.append(f"7th Character Guidance: {g['seventh_char_guidance']}")
if g.get("is_valid") is False:
parts.append("VALIDATION STATUS: THIS CODE IS INVALID AND UNTRACEABLE.")
if g.get("excludes1"):
parts.append(f"Excludes1 (mutually exclusive — NEVER code together): {g['excludes1']}")
return "\n".join(parts)
if code in cpt:
g = cpt[code]
parts = [
f"CPT Guideline for {code}: {g['description']}",
f"Official Note: {g['note']}",
]
if g.get("ncci_conflicts"):
parts.append(f"NCCI Conflicts (cannot bill together): {g['ncci_conflicts']}")
return "\n".join(parts)
return (
f"CODE NOT FOUND: '{code}' could not be located in the ICD-10-CM or CPT "
f"guideline database. This code is untraceable. "
f"Flag it using flag_error with error_type='untraceable_code'."
)
def _lookup_ncci_edit(self, code1: str, code2: str) -> tuple[str, bool]:
"""Check NCCI PTP edit table for a pair of codes. Returns (result_text, found_edit)."""
ncci_edits = self._data.get("ncci_edits", [])
for edit in ncci_edits:
c1, c2 = edit["column1"], edit["column2"]
if {code1, code2} == {c1, c2}:
return (
f"NCCI PTP EDIT FOUND between {code1} and {code2}:\n"
f"Conflict Type: {edit['conflict_type']}\n"
f"Modifier Allowed: {edit.get('modifier_allowed', 'unknown')}\n"
f"Details: {edit['note']}",
True,
)
return (
f"No NCCI PTP edit found between '{code1}' and '{code2}'. "
f"These codes may be billed together on the same claim without a bundling conflict.",
False,
)
def _was_investigated(self, code: str, error_type: str) -> bool:
"""
Check whether the agent investigated this code before flagging it.
Returns True if:
- For any error_type: agent called query_guideline on the code prior to this flag
- For ncci_edit: agent also called check_ncci_edits with this code in the pair
"""
queried = code in self._codes_queried
if error_type == "ncci_edit":
# Also check if the agent did an NCCI check involving this code
ncci_checked = any(code in pair for pair in self._pairs_checked)
return queried or ncci_checked
return queried
def _thorough_review_bonus(self) -> float:
"""
Bonus for querying all proposed codes before submitting.
Returns 0.05 if all codes were queried, 0.0 otherwise.
"""
all_proposed = set(self._proposed_codes.keys())
all_queried = set(self._codes_queried)
# Consider NCCI-checked codes as "investigated" too
for pair in self._pairs_checked:
all_queried.update(pair.split("|"))
if all_proposed <= all_queried:
return 0.05
return 0.0
def _justification_quality_bonus(self, code: str, justification: str) -> float:
"""
Award a small bonus if the justification contains key terms from the
expected error description. Rewards reasoning quality, not just classification.
Returns bonus in [0.0, 0.05].
"""
expected_errors = self._task.get("expected_errors", [])
for exp in expected_errors:
if exp["code"] != code:
continue
key_terms = exp.get("key_terms", [])
if not key_terms:
return 0.0
justification_lower = justification.lower()
hits = sum(1 for t in key_terms if t.lower() in justification_lower)
# Require at least 2 key terms for the bonus
if hits >= 2:
return 0.05
return 0.0
def _compute_episode_metrics(self, base_score: float, fp_count: int) -> Dict[str, Any]:
"""
Feature M: Compute trajectory evaluation metrics at episode end.
Returns a dict with metrics useful for RL training diagnostics:
- trajectory_length: total steps taken
- tool_failure_rate: fraction of steps that returned an error
- investigation_coverage: fraction of proposed codes investigated
- flag_precision: fraction of flags that are true positives
- flag_recall: fraction of expected errors that were flagged correctly
- avg_reward_per_step: mean per-step reward (excluding terminal)
- investigation_before_flag_rate: fraction of flags preceded by investigation
- clarification_count: number of clarifying questions asked
"""
steps = self._state.step_count
expected_errors = self._task.get("expected_errors", [])
expected_by_code = {e["code"]: e["error_type"] for e in expected_errors}
# Tool failure rate: actions with an error response
error_actions = sum(
1 for atype, _ in self._action_history
if atype in ("query_guideline", "check_ncci_edits", "flag_error", "ask_clarifying_question")
)
# Count repeated/hallucinated actions as failures by checking action_history length
failure_steps = sum(
1 for atype, val in self._action_history
if val is not None and atype == "query_guideline" and (
val not in self._proposed_codes or self._action_history.count((atype, val)) > 1
)
)
tool_failure_rate = round(failure_steps / max(steps, 1), 4)
# Investigation coverage
investigated: Set[str] = set(self._codes_queried)
for pair in self._pairs_checked:
investigated.update(pair.split("|"))
proposed_set = set(self._proposed_codes.keys())
investigation_coverage = round(
len(investigated & proposed_set) / max(len(proposed_set), 1), 4
)
# Flag precision / recall
true_positives = sum(
1 for e in self._draft_report
if e["code"] in expected_by_code and e["error_type"] == expected_by_code[e["code"]]
)
flag_precision = round(true_positives / max(len(self._draft_report), 1), 4)
flag_recall = round(true_positives / max(len(expected_errors), 1), 4)
# Investigation-before-flag rate
flag_actions = [
val for atype, val in self._action_history if atype == "flag_error" and val
]
investigated_before_flag = sum(
1 for code in flag_actions if code in investigated
)
investigation_before_flag_rate = round(
investigated_before_flag / max(len(flag_actions), 1), 4
)
return {
"trajectory_length": steps,
"tool_failure_rate": tool_failure_rate,
"investigation_coverage": investigation_coverage,
"flag_precision": flag_precision,
"flag_recall": flag_recall,
"base_grader_score": round(base_score, 4),
"false_positive_count": fp_count,
"investigation_before_flag_rate": investigation_before_flag_rate,
"clarification_count": len(self._clarifications_asked),
}
def _score_flag(
self,
code: str,
error_type: str,
justification: str,
expected_errors: Dict[str, str],
) -> tuple[float, str]:
"""
Assign per-step reward for a flag_error action.
Returns (reward, feedback_message).
"""
if code not in expected_errors:
# False positive — this code has no expected error
return -0.2, (
f"Warning: No coding error is expected for '{code}' in this case. "
f"This may be a false positive. Review the guidelines again."
)
correct_error_type = expected_errors[code]
if error_type == correct_error_type:
# Correct code AND correct error_type
justification_bonus = self._justification_quality_bonus(code, justification)
# Feature H: evidence extraction bonus
evidence_bonus = 0.05 if code in self._evidence_covered_codes else 0.0
# Feature L: rarity multiplier scales the base reward
rarity = _ERROR_RARITY.get(error_type, 1.0)
if error_type in ("ncci_edit", "excludes1_conflict"):
base = round(0.3 * rarity, 3)
return base + justification_bonus + evidence_bonus, (
f"Correct! '{code}' has a '{error_type}' error. "
f"Full reward for NCCI/Excludes1 conflict identification (rarity×{rarity})."
+ (f" Justification bonus: +{justification_bonus:.2f}" if justification_bonus > 0 else "")
+ (f" Evidence bonus: +{evidence_bonus:.2f}" if evidence_bonus > 0 else "")
)
else:
base = round(0.2 * rarity, 3)
return base + justification_bonus + evidence_bonus, (
f"Correct! '{code}' has a '{error_type}' error. "
f"Reward for accurate error classification (rarity×{rarity})."
+ (f" Justification bonus: +{justification_bonus:.2f}" if justification_bonus > 0 else "")
+ (f" Evidence bonus: +{evidence_bonus:.2f}" if evidence_bonus > 0 else "")
)
else:
# Correct code but wrong error_type — HERON-scaled penalty by conceptual distance
distance = _ERROR_TYPE_DISTANCE.get((error_type, correct_error_type), 2)
penalty = -0.05 * distance # -0.05 for close miss, -0.10 for far miss
return penalty, (
f"Code '{code}' does have an error, but '{error_type}' does not match "
f"the expected classification (distance={distance}). "
f"Re-examine the coding guidelines for this code."
)
def _build_observation(
self,
tool_result: str,
reward: float,
done: bool,
error: Optional[str] = None,
) -> MedicalCodingObservation:
"""Construct a full observation from current environment state."""
if self._task is None:
# Before reset() is called
return MedicalCodingObservation(
task_id="",
difficulty="",
patient_demographics={},
clinical_note="",
proposed_codes={},
draft_report=[],
tool_result=tool_result,
step_count=self._state.step_count,
codes_queried=self._codes_queried,
pairs_checked=self._pairs_checked,
clarifications_asked=self._clarifications_asked,
extracted_evidence=self._extracted_evidence,
last_action_error=error,
grader_score=self._grader_score,
episode_metrics=self._episode_metrics,
reward=reward,
done=done,
)
# Partial observability: show only revealed sections
if self._partial_observability and self._note_sections:
visible_note = _build_visible_note(self._note_sections, self._revealed_section_count)
else:
visible_note = self._task["clinical_note"]
return MedicalCodingObservation(
task_id=self._task["task_id"],
difficulty=self._task["difficulty"],
patient_demographics=self._task["patient"],
clinical_note=visible_note,
proposed_codes=self._proposed_codes,
draft_report=list(self._draft_report),
tool_result=tool_result,
step_count=self._state.step_count,
codes_queried=list(self._codes_queried),
pairs_checked=list(self._pairs_checked),
clarifications_asked=list(self._clarifications_asked),
extracted_evidence=list(self._extracted_evidence),
last_action_error=error,
grader_score=self._grader_score,
episode_metrics=self._episode_metrics,
reward=reward,
done=done,
)
|