File size: 48,782 Bytes
3863567
 
 
 
 
 
 
1da8d42
3863567
 
 
 
1da8d42
8667e08
 
3863567
1da8d42
 
3863567
1da8d42
 
8667e08
27697e4
 
 
1da8d42
8667e08
27697e4
 
 
 
 
8667e08
 
 
 
 
3863567
 
 
 
 
 
8667e08
3863567
8667e08
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27697e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8667e08
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3863567
 
 
 
 
 
 
1da8d42
3863567
 
 
 
 
1da8d42
3863567
1da8d42
 
 
3863567
1da8d42
3863567
 
1da8d42
3863567
1da8d42
 
 
 
3863567
 
 
 
 
1da8d42
 
3863567
 
 
 
 
 
 
 
 
 
 
27697e4
 
3863567
27697e4
3863567
27697e4
 
 
 
 
 
 
 
3863567
27697e4
 
 
1da8d42
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27697e4
 
 
 
 
 
3863567
 
 
 
8667e08
3863567
8667e08
3863567
 
 
 
 
 
 
 
 
 
 
 
 
8667e08
 
 
 
 
 
27697e4
 
 
 
 
 
 
3863567
 
 
 
 
8667e08
3863567
 
 
 
8667e08
 
 
 
3863567
 
8667e08
3863567
 
 
 
 
 
 
8667e08
27697e4
 
 
 
3863567
 
 
8667e08
 
27697e4
 
 
 
 
 
8667e08
 
 
 
3863567
 
 
 
 
 
 
 
8667e08
 
 
 
 
 
 
 
 
 
 
3863567
 
 
 
 
 
8667e08
3863567
 
 
 
 
 
 
 
 
 
1da8d42
3863567
 
 
 
 
 
 
 
 
 
 
1da8d42
 
 
 
3863567
 
 
 
 
27697e4
 
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8667e08
3863567
 
 
 
 
 
 
8667e08
 
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8667e08
 
 
 
3863567
 
 
 
 
 
 
 
 
8667e08
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8667e08
 
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1da8d42
3863567
8667e08
 
 
 
 
 
 
 
 
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1da8d42
 
 
 
 
 
27697e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1da8d42
 
 
 
 
 
 
 
 
 
 
3863567
1da8d42
 
 
 
 
 
 
 
 
 
 
 
8667e08
 
 
27697e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1da8d42
 
 
3863567
 
27697e4
 
 
 
 
 
 
 
3863567
 
 
 
1da8d42
 
8667e08
27697e4
1da8d42
 
 
3863567
 
 
 
 
 
 
 
 
 
 
1da8d42
 
 
 
3863567
 
 
1da8d42
3863567
 
 
 
 
 
 
27697e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8667e08
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1da8d42
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27697e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3863567
 
 
 
1da8d42
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1da8d42
27697e4
 
 
 
3863567
27697e4
 
3863567
27697e4
 
 
3863567
 
27697e4
 
3863567
27697e4
 
 
3863567
 
27697e4
 
 
 
 
 
3863567
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27697e4
 
3863567
 
27697e4
3863567
 
 
 
8667e08
 
 
 
 
 
3863567
 
 
 
8667e08
3863567
 
 
 
 
 
27697e4
 
3863567
 
27697e4
3863567
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
"""
Medical Coding Auditor Environment — Core Logic.

The agent plays the role of a Medical Coding Auditor performing a pre-bill review.
It validates a proposed set of ICD-10 and CPT codes against a clinical note and
patient demographics without assigning new codes.

Reward shaping (dense, multi-signal):
  +0.10  QueryGuideline on a code present in the proposed set
  +0.30  FlagError that correctly identifies an NCCI edit or Excludes1 conflict
  +0.20  FlagError that correctly identifies a demographic mismatch,
         specificity error, or untraceable code
  +0.05  Justification quality bonus (key terms from expected error description)
  +0.05  Investigation coherence bonus (queried code's guideline before flagging it)
  -0.05  Flag without investigation penalty (flagging a code without querying first)
  -0.10  Repeated QueryGuideline / CheckNCCIEdits on an already-queried code/pair
  -0.10  FlagError with correct code but wrong error_type classification
  -0.20  FlagError on a code with no expected error (false positive)
  -0.50  Any action that references a code NOT in the proposed set (hallucination)
  +final Terminal reward on SubmitAudit:
         grader_score - 0.15 * false_positives + 0.1 * (max_steps - steps) / max_steps
         + 0.05 thorough_review bonus (queried all proposed codes before submitting)
  +0.15  Relevant clarifying question (matches trigger keywords in clarification pool)
  -0.10  Irrelevant or repeated clarifying question
  +0.10  Terminal bonus for asking necessary clarifications before flagging
  Auto-termination: episode ends with auto-submit and -0.1 penalty at max_steps

Hierarchical reward (HERON-style):
  Wrong error_type penalty scaled by conceptual distance between error types.
  Close misclassifications (e.g. excludes1↔ncci_edit) penalized less than
  distant ones (e.g. demographic_mismatch↔untraceable_code).

Partial observability mode:
  When enabled via reset(partial_observability=True), the clinical note is
  progressively revealed as the agent queries codes. Initially only the
  REASON FOR VISIT section is visible. Each query_guideline reveals the
  next section. This models real-world information gathering.
"""

from __future__ import annotations

import json
import os
import re
from pathlib import Path
from typing import Any, Dict, List, Optional, Set
from uuid import uuid4

from openenv.core.env_server.interfaces import Environment
from openenv.core.env_server.types import State

try:
    from ..models import MedicalCodingAction, MedicalCodingObservation
except ImportError:
    try:
        from models import MedicalCodingAction, MedicalCodingObservation
    except ImportError:
        from medical_coding_env.models import MedicalCodingAction, MedicalCodingObservation

# ---------------------------------------------------------------------------
# Locate the ground truth data file
# ---------------------------------------------------------------------------
_DATA_FILE = Path(__file__).parent.parent / "data" / "ground_truth_cases.json"


def _load_data() -> Dict[str, Any]:
    with open(_DATA_FILE, "r", encoding="utf-8") as fh:
        return json.load(fh)


# ---------------------------------------------------------------------------
# Feature I: HERON-style error_type distance matrix
# ---------------------------------------------------------------------------
# Distance 0 = same type, 1 = close (same category), 2 = far (different category)
# Categories: "conflict" (excludes1, ncci_edit), "validity" (specificity, untraceable),
#             "demographic" (demographic_mismatch)

_ERROR_TYPE_DISTANCE: Dict[tuple[str, str], int] = {}
_ERROR_TYPES = [
    "demographic_mismatch", "excludes1_conflict", "ncci_edit",
    "specificity_error", "untraceable_code",
]
_ERROR_CATEGORIES = {
    "excludes1_conflict": "conflict",
    "ncci_edit": "conflict",
    "specificity_error": "validity",
    "untraceable_code": "validity",
    "demographic_mismatch": "demographic",
}

for _a in _ERROR_TYPES:
    for _b in _ERROR_TYPES:
        if _a == _b:
            _ERROR_TYPE_DISTANCE[(_a, _b)] = 0
        elif _ERROR_CATEGORIES[_a] == _ERROR_CATEGORIES[_b]:
            _ERROR_TYPE_DISTANCE[(_a, _b)] = 1
        else:
            _ERROR_TYPE_DISTANCE[(_a, _b)] = 2


# ---------------------------------------------------------------------------
# Feature L: Frequency-weighted asymmetric reward multipliers
# ---------------------------------------------------------------------------
# Rarity reflects how often each error type occurs in real-world billing audits.
# Rare, subtle errors are disproportionately rewarded (ASL-inspired).
_ERROR_RARITY: Dict[str, float] = {
    "demographic_mismatch": 1.0,   # common, visually obvious
    "ncci_edit": 1.2,              # moderate — requires NCCI lookup
    "excludes1_conflict": 1.2,     # moderate — requires guideline knowledge
    "untraceable_code": 1.3,       # less common — requires code validation
    "specificity_error": 1.5,      # rare, subtle — 7th character nuance
}

# ---------------------------------------------------------------------------
# Feature K: Adaptive curriculum learning — class-level state
# ---------------------------------------------------------------------------
_CURRICULUM_LEVELS = ["easy", "medium", "hard", "expert"]
_CURRICULUM_PROMOTE_THRESHOLD = 0.8   # avg score to move up
_CURRICULUM_DEMOTE_THRESHOLD = 0.3    # avg score to move down
_CURRICULUM_WINDOW = 5                # episodes to average over

# ---------------------------------------------------------------------------
# Partial Observability — Clinical Note Section Parser
# ---------------------------------------------------------------------------

# Common clinical note section headers
_SECTION_PATTERN = re.compile(
    r"^(REASON FOR VISIT|HISTORY OF PRESENT ILLNESS|PAST MEDICAL HISTORY|"
    r"SOCIAL HISTORY|REVIEW OF SYSTEMS|MEDICATIONS|PHYSICAL EXAM|"
    r"EXAMINATION|IMAGING|SPIROMETRY[^:]*|SERVICES RENDERED|"
    r"ENDOCRINOLOGY ASSESSMENT|CARDIOLOGY ASSESSMENT|RIGHT ARM|"
    r"ASSESSMENT AND PLAN|ASSESSMENT|PLAN)\s*:",
    re.MULTILINE | re.IGNORECASE,
)


def _parse_note_sections(clinical_note: str) -> List[tuple[str, str]]:
    """
    Parse a clinical note into (header, body) sections.
    Returns at least one section. Unparseable notes return as a single section.
    """
    matches = list(_SECTION_PATTERN.finditer(clinical_note))
    if not matches:
        return [("CLINICAL NOTE", clinical_note)]

    sections: List[tuple[str, str]] = []
    for i, match in enumerate(matches):
        header = match.group(1).upper()
        start = match.end()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(clinical_note)
        body = clinical_note[start:end].strip()
        sections.append((header, body))

    return sections


def _build_visible_note(
    sections: List[tuple[str, str]],
    revealed_count: int,
) -> str:
    """Build the currently visible clinical note from revealed sections."""
    if not sections:
        return ""
    visible = sections[:revealed_count]
    hidden_count = len(sections) - revealed_count
    parts = [f"{header}: {body}" for header, body in visible]
    if hidden_count > 0:
        parts.append(f"\n[{hidden_count} more section(s) not yet revealed — query codes to investigate further]")
    return "\n\n".join(parts)


# ---------------------------------------------------------------------------
# Deterministic Grader
# ---------------------------------------------------------------------------

def _grade_audit(
    draft_report: List[Dict[str, Any]],
    expected_errors: List[Dict[str, Any]],
) -> tuple[float, int]:
    """
    Compare the agent's submitted audit report against the ground truth.

    Scoring logic:
    - For each expected error, check if the agent correctly flagged it.
    - A flag is "correct" if both the code AND error_type match → 1.0 credit.
    - Partial credit (0.5x) if the correct code is flagged but with a wrong error_type.
    - False positives (flags on codes with no expected error) are counted and penalized
      in the terminal reward calculation.
    - Final base score = sum(credit for each expected error) / len(expected_errors)

    Returns (base_score, false_positive_count).
    """
    if not expected_errors:
        return 1.0, 0

    expected_codes = {e["code"] for e in expected_errors}

    flagged_by_code: Dict[str, List[str]] = {}
    false_positives = 0
    for entry in draft_report:
        code = entry.get("code", "")
        if code not in flagged_by_code:
            flagged_by_code[code] = []
        flagged_by_code[code].append(entry.get("error_type", ""))
        if code not in expected_codes:
            false_positives += 1

    total_credit = 0.0
    for expected in expected_errors:
        exp_code = expected["code"]
        exp_error_type = expected["error_type"]

        if exp_code not in flagged_by_code:
            continue

        agent_error_types = flagged_by_code[exp_code]

        # Feature L: rarity multiplier applied to grader credit
        rarity = _ERROR_RARITY.get(exp_error_type, 1.0)
        if exp_error_type in agent_error_types:
            total_credit += 1.0 * rarity
        else:
            # HERON: partial credit scaled by conceptual distance of best agent guess
            best_distance = min(
                _ERROR_TYPE_DISTANCE.get((atype, exp_error_type), 2)
                for atype in agent_error_types
            )
            # distance=1 (close miss) → 0.55 credit, distance=2 (far miss) → 0.4 credit
            partial = max(0.4, 0.7 - 0.15 * best_distance)
            total_credit += partial * rarity

    # Normalize by total possible credit (sum of rarity weights, not raw count)
    max_credit = sum(_ERROR_RARITY.get(e["error_type"], 1.0) for e in expected_errors)
    base_score = round(total_credit / max(max_credit, 1.0), 4)
    return base_score, false_positives


# ---------------------------------------------------------------------------
# Environment
# ---------------------------------------------------------------------------

class MedicalCodingEnvironment(Environment):
    """
    Medical Coding Auditor OpenEnv Environment.

    The agent reviews a case containing patient demographics, a clinical note,
    and a proposed set of ICD-10-CM / CPT billing codes. It uses four tools:
      - query_guideline: fetch coding rules for a specific code
      - check_ncci_edits: check for CMS NCCI PTP bundling conflicts
      - flag_error: add an entry to the draft audit report
      - submit_audit: end the episode and trigger the grader

    Task selection: pass task_id as a keyword argument to reset().
    Available task_ids: easy_demographic, medium_ncci_conflict, hard_specificity_untraceable

    The environment supports concurrent WebSocket sessions.
    """

    SUPPORTS_CONCURRENT_SESSIONS: bool = True

    # Feature K: class-level curriculum state (shared across episodes)
    _performance_history: Dict[str, List[float]] = {
        level: [] for level in _CURRICULUM_LEVELS
    }
    _curriculum_level: str = "easy"

    # Default task cycling order for sequential inference runs
    _TASK_ORDER = [
        "easy_demographic",
        "medium_ncci_conflict",
        "medium_excludes1",
        "hard_specificity_untraceable",
        "expert_multi_error",
    ]

    def __init__(self) -> None:
        self._data = _load_data()
        self._state = State(episode_id=str(uuid4()), step_count=0)
        self._task: Optional[Dict[str, Any]] = None
        self._proposed_codes: Dict[str, Dict[str, str]] = {}
        self._draft_report: List[Dict[str, Any]] = []
        self._codes_queried: List[str] = []
        self._pairs_checked: List[str] = []
        self._done: bool = False
        self._grader_score: Optional[float] = None
        self._max_steps: int = 20
        # Feature F: investigation strategy tracking
        self._action_history: List[tuple[str, Optional[str]]] = []
        # Feature E: partial observability
        self._partial_observability: bool = False
        self._note_sections: List[tuple[str, str]] = []  # (header, body) pairs
        self._revealed_section_count: int = 0
        # Feature G: physician clarifications
        self._clarifications_asked: List[str] = []
        # Feature H: evidence extraction
        self._extracted_evidence: List[str] = []        # all extracted spans
        self._evidence_covered_codes: Set[str] = set()  # codes with relevant evidence
        # Feature M: episode metrics (populated at episode end)
        self._episode_metrics: Optional[Dict[str, Any]] = None

    # ------------------------------------------------------------------
    # Public API
    # ------------------------------------------------------------------

    def reset(self, task_id: Optional[str] = None, partial_observability: Optional[bool] = None, **kwargs: Any) -> MedicalCodingObservation:  # type: ignore[override]
        """
        Reset the environment for a new episode.

        Args:
            task_id: Which task to load. If None, cycles through tasks in order.
                     Use 'random' for a procedurally generated case.
            partial_observability: If True, the clinical note is progressively
                     revealed as the agent queries codes. Default: False.

        Returns:
            Initial MedicalCodingObservation with the case context.
        """
        self._state = State(episode_id=str(uuid4()), step_count=0)
        self._draft_report = []
        self._codes_queried = []
        self._pairs_checked = []
        self._done = False
        self._grader_score = None
        self._action_history = []
        self._clarifications_asked = []
        self._extracted_evidence = []
        self._evidence_covered_codes = set()
        self._episode_metrics = None

        # Select task
        tasks = {t["task_id"]: t for t in self._data["tasks"]}
        if task_id and task_id.startswith("random"):
            from data.case_generator import generate_random_task
            if "_" in task_id:
                # Explicit difficulty suffix: random_easy, random_medium, etc.
                difficulty = task_id.split("_", 1)[1]
            else:
                # Feature K: no suffix → use adaptive curriculum level
                difficulty = self._update_curriculum()
            self._task = generate_random_task(
                self._data, self._state.episode_id, difficulty=difficulty,
            )
        elif task_id and task_id in tasks:
            self._task = tasks[task_id]
        else:
            idx = hash(self._state.episode_id) % len(self._TASK_ORDER)
            self._task = tasks[self._TASK_ORDER[idx]]

        self._proposed_codes = self._task["proposed_codes"]
        self._max_steps = self._task["max_steps"]

        # Partial observability setup
        self._partial_observability = bool(partial_observability)
        if self._partial_observability:
            self._note_sections = _parse_note_sections(self._task["clinical_note"])
            self._revealed_section_count = 1  # start with just REASON FOR VISIT
        else:
            self._note_sections = []
            self._revealed_section_count = 0

        po_note = " [PARTIAL OBSERVABILITY: clinical note will be revealed as you investigate.]" if self._partial_observability else ""

        return self._build_observation(
            tool_result=(
                f"Audit session started. Task: {self._task['task_id']} "
                f"({self._task['difficulty']} difficulty). "
                f"Review the patient demographics, clinical note, and proposed codes. "
                f"Use query_guideline, check_ncci_edits, flag_error, then submit_audit."
                f"{po_note}"
            ),
            reward=0.0,
            done=False,
        )

    def step(self, action: MedicalCodingAction) -> MedicalCodingObservation:  # type: ignore[override]
        """
        Execute one auditor action and return the resulting observation.

        Dense reward shaping is applied per the specification.
        Auto-terminates at max_steps with a timeout penalty.
        """
        if self._done:
            return self._build_observation(
                tool_result="Episode already ended. Call reset() to start a new audit.",
                reward=0.0,
                done=True,
                error="Episode is done. Call reset().",
            )

        self._state.step_count += 1

        # Auto-termination: if this is the last allowed step, force submit
        if self._state.step_count >= self._max_steps and action.action_type != "submit_audit":
            return self._handle_auto_termination()

        # Route to the appropriate handler
        handler = {
            "query_guideline": self._handle_query_guideline,
            "check_ncci_edits": self._handle_check_ncci_edits,
            "flag_error": self._handle_flag_error,
            "ask_clarifying_question": self._handle_ask_clarification,
            "extract_evidence": self._handle_extract_evidence,
            "submit_audit": self._handle_submit_audit,
        }.get(action.action_type)

        if handler is None:
            return self._build_observation(
                tool_result=f"Unknown action_type: {action.action_type}",
                reward=-0.2,
                done=False,
                error=f"Unknown action_type: {action.action_type}",
            )

        return handler(action)

    @property
    def state(self) -> State:
        return self._state

    # ------------------------------------------------------------------
    # Action handlers
    # ------------------------------------------------------------------

    def _handle_query_guideline(self, action: MedicalCodingAction) -> MedicalCodingObservation:
        """Look up coding guidelines for a specific code."""
        code = action.code
        if not code:
            self._action_history.append(("query_guideline", None))
            return self._build_observation(
                tool_result="query_guideline requires a 'code' field.",
                reward=-0.2,
                done=False,
                error="Missing required field: code",
            )

        self._action_history.append(("query_guideline", code))

        # Hallucination check — code must be in the proposed set
        if code not in self._proposed_codes:
            return self._build_observation(
                tool_result=(
                    f"Code '{code}' is not in the proposed code set for this encounter. "
                    f"Proposed codes are: {list(self._proposed_codes.keys())}. "
                    f"Only query codes that appear in the proposed billing set."
                ),
                reward=-0.5,
                done=False,
                error=f"Hallucinated code: '{code}' is not in the proposed code set.",
            )

        # Repeated query penalty
        if code in self._codes_queried:
            return self._build_observation(
                tool_result=f"You have already queried guidelines for '{code}' this session. Check your draft report.",
                reward=-0.1,
                done=False,
                error=f"Repeated query: '{code}' already queried.",
            )

        # Look up guideline from data
        guideline = self._lookup_guideline(code)
        self._codes_queried.append(code)

        # Partial observability: reveal next section
        if self._partial_observability and self._revealed_section_count < len(self._note_sections):
            self._revealed_section_count += 1

        return self._build_observation(
            tool_result=guideline,
            reward=0.1,
            done=False,
        )

    def _handle_check_ncci_edits(self, action: MedicalCodingAction) -> MedicalCodingObservation:
        """Check CMS NCCI PTP edit table for two codes."""
        code1, code2 = action.code1, action.code2
        self._action_history.append(("check_ncci_edits", f"{code1}|{code2}"))
        if not code1 or not code2:
            return self._build_observation(
                tool_result="check_ncci_edits requires both 'code1' and 'code2' fields.",
                reward=-0.2,
                done=False,
                error="Missing required field(s): code1 and/or code2",
            )

        # Hallucination check — both codes must be in the proposed set
        hallucinated = [c for c in (code1, code2) if c not in self._proposed_codes]
        if hallucinated:
            return self._build_observation(
                tool_result=(
                    f"Code(s) {hallucinated} are not in the proposed code set. "
                    f"Only check NCCI edits for codes in the proposed billing set: "
                    f"{list(self._proposed_codes.keys())}."
                ),
                reward=-0.5,
                done=False,
                error=f"Hallucinated code(s): {hallucinated}",
            )

        # Repeated check penalty
        pair_key = "|".join(sorted([code1, code2]))
        if pair_key in self._pairs_checked:
            return self._build_observation(
                tool_result=f"You have already checked the NCCI edit between '{code1}' and '{code2}'.",
                reward=-0.1,
                done=False,
                error=f"Repeated NCCI check: {pair_key}",
            )

        # Look up NCCI edit
        ncci_result, found_edit = self._lookup_ncci_edit(code1, code2)
        self._pairs_checked.append(pair_key)

        # Small positive reward for checking a valid pair (no hallucination)
        reward = 0.05 if not found_edit else 0.1

        return self._build_observation(
            tool_result=ncci_result,
            reward=reward,
            done=False,
        )

    def _handle_flag_error(self, action: MedicalCodingAction) -> MedicalCodingObservation:
        """Record a coding error in the draft audit report."""
        code = action.code
        error_type = action.error_type
        justification = action.justification

        self._action_history.append(("flag_error", code))

        if not code or not error_type or not justification:
            missing = [f for f, v in [("code", code), ("error_type", error_type), ("justification", justification)] if not v]
            return self._build_observation(
                tool_result=f"flag_error requires: code, error_type, justification. Missing: {missing}",
                reward=-0.2,
                done=False,
                error=f"Missing required field(s): {missing}",
            )

        # Hallucination check — code must be in the proposed set
        if code not in self._proposed_codes:
            return self._build_observation(
                tool_result=(
                    f"Cannot flag '{code}' — it is not in the proposed code set. "
                    f"Only flag codes that appear in the proposed billing set: "
                    f"{list(self._proposed_codes.keys())}."
                ),
                reward=-0.5,
                done=False,
                error=f"Hallucinated code: '{code}' is not in the proposed code set.",
            )

        # Repeated flag check
        already_flagged = any(
            e["code"] == code and e["error_type"] == error_type
            for e in self._draft_report
        )
        if already_flagged:
            return self._build_observation(
                tool_result=f"Code '{code}' has already been flagged as '{error_type}'. Avoid duplicate flags.",
                reward=-0.1,
                done=False,
                error=f"Duplicate flag: '{code}' / '{error_type}'",
            )

        # Evaluate correctness against ground truth
        expected_errors = {e["code"]: e["error_type"] for e in self._task["expected_errors"]}
        reward, feedback = self._score_flag(code, error_type, justification, expected_errors)

        # Feature F: investigation coherence bonus/penalty
        investigated = self._was_investigated(code, error_type)
        if investigated:
            reward += 0.05
            feedback += " (Investigation coherence bonus: +0.05)"
        else:
            reward -= 0.05
            feedback += " (No prior investigation of this code: -0.05)"

        # Record in draft report
        self._draft_report.append({
            "code": code,
            "error_type": error_type,
            "justification": justification,
            "step": self._state.step_count,
        })

        return self._build_observation(
            tool_result=f"Error flagged for '{code}' as '{error_type}'. {feedback}",
            reward=reward,
            done=False,
        )

    def _handle_submit_audit(self, action: MedicalCodingAction) -> MedicalCodingObservation:
        """End the episode and grade the submitted audit report."""
        return self._finalize_episode(timeout=False)

    def _handle_auto_termination(self) -> MedicalCodingObservation:
        """Auto-submit when max_steps is reached. Applies a timeout penalty."""
        return self._finalize_episode(timeout=True)

    def _handle_ask_clarification(self, action: MedicalCodingAction) -> MedicalCodingObservation:
        """
        Simulate asking the physician a clarifying question.

        The environment matches the question against keyword triggers in the task's
        clarification_pool. A relevant question yields a physician response and a
        +0.15 reward. Irrelevant or repeated questions are penalized.
        """
        question = action.question
        self._action_history.append(("ask_clarifying_question", question))

        if not question:
            return self._build_observation(
                tool_result="ask_clarifying_question requires a 'question' field.",
                reward=-0.2,
                done=False,
                error="Missing required field: question",
            )

        # Repeated question penalty
        if question in self._clarifications_asked:
            return self._build_observation(
                tool_result="You have already asked this question. Avoid repeating clarifications.",
                reward=-0.1,
                done=False,
                error="Repeated clarifying question.",
            )

        self._clarifications_asked.append(question)

        # Match against clarification_pool (keyword-based)
        pool = self._task.get("clarification_pool", [])
        question_lower = question.lower()

        for entry in pool:
            triggers = entry.get("triggers", [])
            if any(t.lower() in question_lower for t in triggers):
                response = entry["response"]
                return self._build_observation(
                    tool_result=(
                        f"PHYSICIAN RESPONSE: {response}"
                    ),
                    reward=0.15,
                    done=False,
                )

        # No matching clarification — irrelevant question
        return self._build_observation(
            tool_result=(
                "PHYSICIAN RESPONSE: I don't have additional information on that. "
                "Please proceed with the available documentation."
            ),
            reward=-0.1,
            done=False,
            error="Irrelevant clarifying question (no matching trigger in task pool).",
        )

    def _handle_extract_evidence(self, action: MedicalCodingAction) -> MedicalCodingObservation:
        """
        Feature H: Extract a text span from the clinical note as evidence.

        Validates the span exists verbatim in the note (hallucination guard).
        Checks if it overlaps with expected evidence spans — rewards grounded reasoning.
        Tracks which codes have documented evidence for the flag_error bonus.
        """
        span = action.evidence_text
        self._action_history.append(("extract_evidence", span))

        if not span:
            return self._build_observation(
                tool_result="extract_evidence requires an 'evidence_text' field.",
                reward=-0.2,
                done=False,
                error="Missing required field: evidence_text",
            )

        clinical_note = self._task["clinical_note"]

        # Hallucination guard: span must appear verbatim in the note
        if span not in clinical_note:
            return self._build_observation(
                tool_result=(
                    f"Evidence text not found verbatim in the clinical note. "
                    f"Only extract exact substrings from the note."
                ),
                reward=-0.05,
                done=False,
                error="Hallucinated evidence: span not found in clinical note.",
            )

        # Repeated extraction penalty
        if span in self._extracted_evidence:
            return self._build_observation(
                tool_result="This evidence span has already been extracted.",
                reward=-0.05,
                done=False,
                error="Repeated evidence extraction.",
            )

        self._extracted_evidence.append(span)
        span_lower = span.lower()

        # Check against expected error evidence spans
        matched_codes = []
        for exp in self._task.get("expected_errors", []):
            for evidence_span in exp.get("evidence_spans", []):
                if evidence_span.lower() in span_lower or span_lower in evidence_span.lower():
                    matched_codes.append(exp["code"])
                    self._evidence_covered_codes.add(exp["code"])
                    break

        if matched_codes:
            return self._build_observation(
                tool_result=(
                    f"Evidence extracted: \"{span[:120]}\"\n"
                    f"This span is relevant to coding decision(s) for: {matched_codes}."
                ),
                reward=0.05,
                done=False,
            )

        # Valid span but not linked to any expected error
        return self._build_observation(
            tool_result=(
                f"Evidence extracted: \"{span[:120]}\"\n"
                f"This span does not appear to be directly relevant to the expected coding errors. "
                f"Continue investigating."
            ),
            reward=0.0,
            done=False,
        )

    def _finalize_episode(self, timeout: bool) -> MedicalCodingObservation:
        """
        Grade the audit and compute the terminal reward.

        Terminal reward = base_grader_score
                        - 0.15 * false_positive_count
                        + efficiency_bonus (0.1 * remaining_steps / max_steps)
                        - timeout_penalty (0.1 if auto-terminated)

        Clamped to [0.0, 1.0].
        """
        expected_errors = self._task["expected_errors"]
        base_score, fp_count = _grade_audit(self._draft_report, expected_errors)

        # Efficiency bonus: reward finishing early
        steps_used = self._state.step_count
        efficiency_bonus = 0.1 * max(self._max_steps - steps_used, 0) / self._max_steps

        # False positive penalty
        fp_penalty = 0.15 * fp_count

        # Timeout penalty
        timeout_penalty = 0.1 if timeout else 0.0

        # Thorough review bonus
        review_bonus = self._thorough_review_bonus() if not timeout else 0.0

        # Feature G: clarification terminal bonus
        # +0.10 if agent asked at least one relevant clarification before flagging
        clarification_bonus = 0.0
        pool = self._task.get("clarification_pool", [])
        if pool and self._clarifications_asked:
            # Check if any asked question matched a pool trigger
            matched = False
            for entry in pool:
                triggers = entry.get("triggers", [])
                for q in self._clarifications_asked:
                    if any(t.lower() in q.lower() for t in triggers):
                        matched = True
                        break
                if matched:
                    break
            if matched:
                clarification_bonus = 0.10

        terminal_reward = (
            base_score
            - fp_penalty
            + efficiency_bonus
            - timeout_penalty
            + review_bonus
            + clarification_bonus
        )
        terminal_reward = round(max(min(terminal_reward, 1.0), 0.0), 4)

        self._grader_score = terminal_reward
        self._done = True

        # Feature K: record score in curriculum history
        difficulty = self._task.get("difficulty", "easy")
        if difficulty in MedicalCodingEnvironment._performance_history:
            MedicalCodingEnvironment._performance_history[difficulty].append(terminal_reward)

        # Feature M: compute episode metrics
        self._episode_metrics = self._compute_episode_metrics(base_score, fp_count)

        errors_found = len(self._draft_report)
        errors_expected = len(expected_errors)

        summary_lines = [
            f"AUDIT {'AUTO-SUBMITTED (max steps reached)' if timeout else 'SUBMITTED'}.",
            f"Base grader score: {base_score:.2f} | Efficiency bonus: +{efficiency_bonus:.2f}"
            + (f" | Thorough review bonus: +{review_bonus:.2f}" if review_bonus > 0 else "")
            + (f" | Clarification bonus: +{clarification_bonus:.2f}" if clarification_bonus > 0 else "")
            + (f" | Timeout penalty: -{timeout_penalty:.2f}" if timeout else "")
            + (f" | False positive penalty: -{fp_penalty:.2f}" if fp_count > 0 else ""),
            f"Final score: {terminal_reward:.2f}",
            f"Errors flagged: {errors_found} | Errors expected: {errors_expected}",
        ]

        for exp in expected_errors:
            flagged = any(
                e["code"] == exp["code"] and e["error_type"] == exp["error_type"]
                for e in self._draft_report
            )
            status = "FOUND" if flagged else "MISSED"
            summary_lines.append(f"  [{status}] {exp['code']} ({exp['error_type']})")

        if fp_count > 0:
            expected_codes = {e["code"] for e in expected_errors}
            fp_entries = [e["code"] for e in self._draft_report if e["code"] not in expected_codes]
            summary_lines.append(f"  False positives: {fp_entries}")

        return self._build_observation(
            tool_result="\n".join(summary_lines),
            reward=terminal_reward,
            done=True,
        )

    # ------------------------------------------------------------------
    # Helpers
    # ------------------------------------------------------------------

    def _update_curriculum(self) -> str:
        """
        Feature K: Adaptive curriculum learning (HiCu-style).

        Reads the class-level performance history to decide what difficulty to use
        for the next random episode. Promotes when avg score > 0.8 over the last
        N episodes; demotes when avg < 0.3. Returns the chosen difficulty string.
        """
        level = MedicalCodingEnvironment._curriculum_level
        history = MedicalCodingEnvironment._performance_history.get(level, [])

        if len(history) >= _CURRICULUM_WINDOW:
            avg = sum(history[-_CURRICULUM_WINDOW:]) / _CURRICULUM_WINDOW
            idx = _CURRICULUM_LEVELS.index(level)
            if avg >= _CURRICULUM_PROMOTE_THRESHOLD and idx < len(_CURRICULUM_LEVELS) - 1:
                MedicalCodingEnvironment._curriculum_level = _CURRICULUM_LEVELS[idx + 1]
            elif avg <= _CURRICULUM_DEMOTE_THRESHOLD and idx > 0:
                MedicalCodingEnvironment._curriculum_level = _CURRICULUM_LEVELS[idx - 1]

        return MedicalCodingEnvironment._curriculum_level

    def _lookup_guideline(self, code: str) -> str:
        """Return official guideline text for a given code."""
        icd10 = self._data.get("icd10_guidelines", {})
        cpt = self._data.get("cpt_guidelines", {})

        if code in icd10:
            g = icd10[code]
            parts = [
                f"ICD-10-CM Guideline for {code}: {g['description']}",
                f"Official Note: {g['note']}",
            ]
            if g.get("gender_restriction"):
                parts.append(f"Gender Restriction: APPLICABLE TO {g['gender_restriction'].upper()} PATIENTS ONLY.")
            if g.get("seventh_char_guidance"):
                parts.append(f"7th Character Guidance: {g['seventh_char_guidance']}")
            if g.get("is_valid") is False:
                parts.append("VALIDATION STATUS: THIS CODE IS INVALID AND UNTRACEABLE.")
            if g.get("excludes1"):
                parts.append(f"Excludes1 (mutually exclusive — NEVER code together): {g['excludes1']}")
            return "\n".join(parts)

        if code in cpt:
            g = cpt[code]
            parts = [
                f"CPT Guideline for {code}: {g['description']}",
                f"Official Note: {g['note']}",
            ]
            if g.get("ncci_conflicts"):
                parts.append(f"NCCI Conflicts (cannot bill together): {g['ncci_conflicts']}")
            return "\n".join(parts)

        return (
            f"CODE NOT FOUND: '{code}' could not be located in the ICD-10-CM or CPT "
            f"guideline database. This code is untraceable. "
            f"Flag it using flag_error with error_type='untraceable_code'."
        )

    def _lookup_ncci_edit(self, code1: str, code2: str) -> tuple[str, bool]:
        """Check NCCI PTP edit table for a pair of codes. Returns (result_text, found_edit)."""
        ncci_edits = self._data.get("ncci_edits", [])

        for edit in ncci_edits:
            c1, c2 = edit["column1"], edit["column2"]
            if {code1, code2} == {c1, c2}:
                return (
                    f"NCCI PTP EDIT FOUND between {code1} and {code2}:\n"
                    f"Conflict Type: {edit['conflict_type']}\n"
                    f"Modifier Allowed: {edit.get('modifier_allowed', 'unknown')}\n"
                    f"Details: {edit['note']}",
                    True,
                )

        return (
            f"No NCCI PTP edit found between '{code1}' and '{code2}'. "
            f"These codes may be billed together on the same claim without a bundling conflict.",
            False,
        )

    def _was_investigated(self, code: str, error_type: str) -> bool:
        """
        Check whether the agent investigated this code before flagging it.

        Returns True if:
        - For any error_type: agent called query_guideline on the code prior to this flag
        - For ncci_edit: agent also called check_ncci_edits with this code in the pair
        """
        queried = code in self._codes_queried
        if error_type == "ncci_edit":
            # Also check if the agent did an NCCI check involving this code
            ncci_checked = any(code in pair for pair in self._pairs_checked)
            return queried or ncci_checked
        return queried

    def _thorough_review_bonus(self) -> float:
        """
        Bonus for querying all proposed codes before submitting.
        Returns 0.05 if all codes were queried, 0.0 otherwise.
        """
        all_proposed = set(self._proposed_codes.keys())
        all_queried = set(self._codes_queried)
        # Consider NCCI-checked codes as "investigated" too
        for pair in self._pairs_checked:
            all_queried.update(pair.split("|"))
        if all_proposed <= all_queried:
            return 0.05
        return 0.0

    def _justification_quality_bonus(self, code: str, justification: str) -> float:
        """
        Award a small bonus if the justification contains key terms from the
        expected error description. Rewards reasoning quality, not just classification.

        Returns bonus in [0.0, 0.05].
        """
        expected_errors = self._task.get("expected_errors", [])
        for exp in expected_errors:
            if exp["code"] != code:
                continue
            key_terms = exp.get("key_terms", [])
            if not key_terms:
                return 0.0
            justification_lower = justification.lower()
            hits = sum(1 for t in key_terms if t.lower() in justification_lower)
            # Require at least 2 key terms for the bonus
            if hits >= 2:
                return 0.05
        return 0.0

    def _compute_episode_metrics(self, base_score: float, fp_count: int) -> Dict[str, Any]:
        """
        Feature M: Compute trajectory evaluation metrics at episode end.

        Returns a dict with metrics useful for RL training diagnostics:
        - trajectory_length: total steps taken
        - tool_failure_rate: fraction of steps that returned an error
        - investigation_coverage: fraction of proposed codes investigated
        - flag_precision: fraction of flags that are true positives
        - flag_recall: fraction of expected errors that were flagged correctly
        - avg_reward_per_step: mean per-step reward (excluding terminal)
        - investigation_before_flag_rate: fraction of flags preceded by investigation
        - clarification_count: number of clarifying questions asked
        """
        steps = self._state.step_count
        expected_errors = self._task.get("expected_errors", [])
        expected_by_code = {e["code"]: e["error_type"] for e in expected_errors}

        # Tool failure rate: actions with an error response
        error_actions = sum(
            1 for atype, _ in self._action_history
            if atype in ("query_guideline", "check_ncci_edits", "flag_error", "ask_clarifying_question")
        )
        # Count repeated/hallucinated actions as failures by checking action_history length
        failure_steps = sum(
            1 for atype, val in self._action_history
            if val is not None and atype == "query_guideline" and (
                val not in self._proposed_codes or self._action_history.count((atype, val)) > 1
            )
        )
        tool_failure_rate = round(failure_steps / max(steps, 1), 4)

        # Investigation coverage
        investigated: Set[str] = set(self._codes_queried)
        for pair in self._pairs_checked:
            investigated.update(pair.split("|"))
        proposed_set = set(self._proposed_codes.keys())
        investigation_coverage = round(
            len(investigated & proposed_set) / max(len(proposed_set), 1), 4
        )

        # Flag precision / recall
        true_positives = sum(
            1 for e in self._draft_report
            if e["code"] in expected_by_code and e["error_type"] == expected_by_code[e["code"]]
        )
        flag_precision = round(true_positives / max(len(self._draft_report), 1), 4)
        flag_recall = round(true_positives / max(len(expected_errors), 1), 4)

        # Investigation-before-flag rate
        flag_actions = [
            val for atype, val in self._action_history if atype == "flag_error" and val
        ]
        investigated_before_flag = sum(
            1 for code in flag_actions if code in investigated
        )
        investigation_before_flag_rate = round(
            investigated_before_flag / max(len(flag_actions), 1), 4
        )

        return {
            "trajectory_length": steps,
            "tool_failure_rate": tool_failure_rate,
            "investigation_coverage": investigation_coverage,
            "flag_precision": flag_precision,
            "flag_recall": flag_recall,
            "base_grader_score": round(base_score, 4),
            "false_positive_count": fp_count,
            "investigation_before_flag_rate": investigation_before_flag_rate,
            "clarification_count": len(self._clarifications_asked),
        }

    def _score_flag(
        self,
        code: str,
        error_type: str,
        justification: str,
        expected_errors: Dict[str, str],
    ) -> tuple[float, str]:
        """
        Assign per-step reward for a flag_error action.

        Returns (reward, feedback_message).
        """
        if code not in expected_errors:
            # False positive — this code has no expected error
            return -0.2, (
                f"Warning: No coding error is expected for '{code}' in this case. "
                f"This may be a false positive. Review the guidelines again."
            )

        correct_error_type = expected_errors[code]
        if error_type == correct_error_type:
            # Correct code AND correct error_type
            justification_bonus = self._justification_quality_bonus(code, justification)
            # Feature H: evidence extraction bonus
            evidence_bonus = 0.05 if code in self._evidence_covered_codes else 0.0
            # Feature L: rarity multiplier scales the base reward
            rarity = _ERROR_RARITY.get(error_type, 1.0)
            if error_type in ("ncci_edit", "excludes1_conflict"):
                base = round(0.3 * rarity, 3)
                return base + justification_bonus + evidence_bonus, (
                    f"Correct! '{code}' has a '{error_type}' error. "
                    f"Full reward for NCCI/Excludes1 conflict identification (rarity×{rarity})."
                    + (f" Justification bonus: +{justification_bonus:.2f}" if justification_bonus > 0 else "")
                    + (f" Evidence bonus: +{evidence_bonus:.2f}" if evidence_bonus > 0 else "")
                )
            else:
                base = round(0.2 * rarity, 3)
                return base + justification_bonus + evidence_bonus, (
                    f"Correct! '{code}' has a '{error_type}' error. "
                    f"Reward for accurate error classification (rarity×{rarity})."
                    + (f" Justification bonus: +{justification_bonus:.2f}" if justification_bonus > 0 else "")
                    + (f" Evidence bonus: +{evidence_bonus:.2f}" if evidence_bonus > 0 else "")
                )
        else:
            # Correct code but wrong error_type — HERON-scaled penalty by conceptual distance
            distance = _ERROR_TYPE_DISTANCE.get((error_type, correct_error_type), 2)
            penalty = -0.05 * distance  # -0.05 for close miss, -0.10 for far miss
            return penalty, (
                f"Code '{code}' does have an error, but '{error_type}' does not match "
                f"the expected classification (distance={distance}). "
                f"Re-examine the coding guidelines for this code."
            )

    def _build_observation(
        self,
        tool_result: str,
        reward: float,
        done: bool,
        error: Optional[str] = None,
    ) -> MedicalCodingObservation:
        """Construct a full observation from current environment state."""
        if self._task is None:
            # Before reset() is called
            return MedicalCodingObservation(
                task_id="",
                difficulty="",
                patient_demographics={},
                clinical_note="",
                proposed_codes={},
                draft_report=[],
                tool_result=tool_result,
                step_count=self._state.step_count,
                codes_queried=self._codes_queried,
                pairs_checked=self._pairs_checked,
                clarifications_asked=self._clarifications_asked,
                extracted_evidence=self._extracted_evidence,
                last_action_error=error,
                grader_score=self._grader_score,
                episode_metrics=self._episode_metrics,
                reward=reward,
                done=done,
            )

        # Partial observability: show only revealed sections
        if self._partial_observability and self._note_sections:
            visible_note = _build_visible_note(self._note_sections, self._revealed_section_count)
        else:
            visible_note = self._task["clinical_note"]

        return MedicalCodingObservation(
            task_id=self._task["task_id"],
            difficulty=self._task["difficulty"],
            patient_demographics=self._task["patient"],
            clinical_note=visible_note,
            proposed_codes=self._proposed_codes,
            draft_report=list(self._draft_report),
            tool_result=tool_result,
            step_count=self._state.step_count,
            codes_queried=list(self._codes_queried),
            pairs_checked=list(self._pairs_checked),
            clarifications_asked=list(self._clarifications_asked),
            extracted_evidence=list(self._extracted_evidence),
            last_action_error=error,
            grader_score=self._grader_score,
            episode_metrics=self._episode_metrics,
            reward=reward,
            done=done,
        )