File size: 72,624 Bytes
46fe844
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
"""Dataset loading, profiling, and feature-matrix construction."""

from __future__ import annotations

import csv
from dataclasses import dataclass
from pathlib import Path
import re
import warnings

import numpy as np
import pandas as pd

from .models import DatasetProfile


class InsufficientDataError(ValueError):
    """Raised when a dataset is too sparse/degenerate for a rigorous run.

    This is an expected guardrail outcome (not a bug): the agent refuses to
    fabricate associations on underpowered, constant, or feature-less data.
    Callers should convert it into a clear user-facing boundary message.
    """


TARGET_HINTS = (
    "target",
    "outcome",
    "label",
    "phq",
    "depression",
    "anxiety",
    "homa",
    "insulin",
    "glucose",
    "severity",
    "score",
)

ID_HINTS = (
    "participant", "subject", "subjid", "usubjid", "user_id", "person", "pid",
    "patient", "record_id", "mrn", "wearer", "studyid", "study_id", "_id",
    "group",    # cluster/group ID (ecological studies, multi-site)
    "cluster",  # cluster ID
    "household", "unit_id",
)
TIME_HINTS = ("time", "date", "timestamp", "datetime", "day", "hour", "week")


@dataclass
class AnalysisFrame:
    frame: pd.DataFrame
    target_column: str
    participant_id_column: str | None
    time_column: str | None
    feature_columns: list[str]
    confounder_columns: list[str]
    excluded_columns: list[str]
    feature_components: dict[str, list[str]]
    audit_log: list[str]


def _detect_delimiter(path: Path) -> str:
    """Detect the field delimiter from the header line (most frequent of , ; tab |). Delimiter
    chars are ASCII, so this is robust to the file's text encoding. Handles European `;` exports,
    TSV, and pipe-delimited files without guessing per-row."""
    try:
        with open(path, encoding="utf-8", errors="replace") as fh:
            line = fh.readline()
    except Exception:
        return ","
    counts = {d: line.count(d) for d in (",", ";", "\t", "|")}
    best = max(counts, key=counts.get)
    return best if counts[best] > 0 else ","


def _robust_read_csv(path: Path, **kwargs) -> pd.DataFrame:
    """Read a CSV the way real users actually upload it: detect the delimiter (`;` European/TSV/
    pipe, not just comma), try common encodings (Excel often exports cp1252/latin-1), handle the
    European comma-decimal convention (`80,5`), and tolerate ragged rows / unbalanced quotes (skip
    bad lines), so a real-world file degrades gracefully instead of crashing. Raises a clear
    InsufficientDataError only when nothing parses at all."""
    explicit_sep = ("sep" in kwargs) or ("delimiter" in kwargs)
    sep = None if explicit_sep else _detect_delimiter(path)

    def _read(enc: str, **extra) -> pd.DataFrame:
        opts = dict(kwargs)
        if sep is not None:
            opts["sep"] = sep
        opts.update(extra)
        return pd.read_csv(path, encoding=enc, **opts)

    def _numeric_cols(df: pd.DataFrame) -> int:
        n = 0
        for col in df.columns:
            s = df[col]
            if pd.api.types.is_numeric_dtype(s) or pd.to_numeric(s, errors="coerce").notna().mean() > 0.8:
                n += 1
        return n

    last_err: Exception | None = None
    for enc in ("utf-8", "utf-8-sig", "cp1252", "latin-1"):
        try:
            base = _read(enc)
        except UnicodeDecodeError as exc:
            last_err = exc
            continue  # wrong encoding -> try the next one
        except pd.errors.ParserError as exc:
            last_err = exc
            py_kwargs = {k: v for k, v in kwargs.items() if k != "low_memory"}
            for extra in ({"on_bad_lines": "skip"}, {"on_bad_lines": "skip", "quoting": csv.QUOTE_NONE}):
                try:
                    return pd.read_csv(path, encoding=enc, sep=(sep or ","), engine="python", **py_kwargs, **extra)
                except (pd.errors.ParserError, UnicodeDecodeError) as exc2:
                    last_err = exc2
                    continue
            continue
        # European comma-decimal: for `;`-delimited files, prefer decimal="," if it yields MORE
        # numeric columns (so "80,5" parses to 80.5 instead of staying a string).
        if sep == ";":
            try:
                euro = _read(enc, decimal=",", thousands=".")
                if _numeric_cols(euro) > _numeric_cols(base):
                    return euro
            except Exception:
                pass
        return base
    raise InsufficientDataError(
        f"Could not parse this CSV ({type(last_err).__name__ if last_err else 'unknown error'}: "
        f"{str(last_err)[:140] if last_err else ''}). Check the delimiter, quoting, and encoding "
        "(UTF-8 or Latin-1) β€” or re-export the file from your tool β€” and upload again."
    )


# Memory guard: above this file size, read a BOUNDED row sample instead of the whole CSV, so a large
# wearable export (which on Cloud Run's in-memory tmpfs would otherwise load fully into RAM) cannot OOM
# the 8Gi instance. The cap is by CELLS (β‰ˆ20M), so wide files read fewer rows. The discovery engine
# further subsamples for screening and discloses it, so analysis on a representative sample stays honest.
_LARGE_FILE_BYTES = 80 * 1024 * 1024
# Keep the in-memory footprint small even for object-heavy files (e.g. wearable exports with dozens of
# repeated string/demographic columns): the engine subsamples to ~40k rows for screening anyway, so a
# ~3M-cell sample is plenty and keeps peak RSS bounded well under the 8Gi instance (which also holds the
# raw file in tmpfs during processing).
_LARGE_FILE_CELL_CAP = 3_000_000
_LARGE_FILE_MIN_ROWS = 20_000
_LARGE_FILE_CHUNK_CELLS = 600_000     # per-chunk parse footprint cap (rows = this // ncols)


def _bounded_read_rows(path: Path) -> int | None:
    """Row cap for a large file (None = read all). Reads only the header to size the cap by columns."""
    try:
        if path.stat().st_size <= _LARGE_FILE_BYTES:
            return None
        ncols = max(1, _robust_read_csv(path, nrows=1).shape[1])
        return max(_LARGE_FILE_MIN_ROWS, _LARGE_FILE_CELL_CAP // ncols)
    except Exception:
        return None


def _count_data_rows(path: Path) -> int:
    """Fast streaming line count (O(1) memory) minus the header row."""
    with open(path, "rb") as f:
        n = sum(buf.count(b"\n") for buf in iter(lambda: f.read(1 << 20), b""))
    return max(1, n - 1)


def _read_csv_systematic_sample(path: Path, cap_rows: int) -> pd.DataFrame:
    """Memory-bounded REPRESENTATIVE read of a large CSV. Streams the file LINE BY LINE (O(1) memory β€”
    pandas' chunked parser still buffers GBs for a multi-GB file), keeps the header + every stride-th
    data line into a small in-memory buffer, then parses only that. Spans the WHOLE file, so all
    participants are represented even when rows are grouped by subject; peak memory β‰ˆ the sample, not
    the file."""
    import io
    total = _count_data_rows(path)
    stride = max(1, total // max(1, cap_rows))
    if stride <= 1:
        return _robust_read_csv(path, low_memory=False)
    buf = io.StringIO()
    try:
        with open(path, "r", newline="", encoding="utf-8", errors="replace") as f:
            header = f.readline()
            if not header:
                return _robust_read_csv(path, nrows=cap_rows, low_memory=False)
            buf.write(header)
            for i, line in enumerate(f):       # i=0 is the FIRST data row
                if i % stride == 0:
                    buf.write(line)
    except Exception:
        return _robust_read_csv(path, nrows=cap_rows, low_memory=False)  # fall back to head sample
    buf.seek(0)
    try:
        # on_bad_lines="skip": real wearable exports are often ragged (a few rows with an extra field).
        return pd.read_csv(buf, low_memory=False, on_bad_lines="skip")
    except Exception:
        return _robust_read_csv(path, nrows=cap_rows, low_memory=False)


def _normalize_string_extension_dtypes(df: pd.DataFrame) -> pd.DataFrame:
    """Coerce pandas 'string'/Arrow-backed string columns (pandas 2.x ``StringDtype`` /
    ``ArrowStringArray``) to plain numpy ``object`` dtype.

    Why: sklearn's array indexing (``train_test_split``, and ``groups=`` for
    ``GroupKFold``/``StratifiedGroupKFold``) cannot integer-index an Arrow-backed array and raises
    ``"only integer scalar arrays can be converted to a scalar index"``. Any dataset with a string
    participant-id column (the common case) would otherwise crash the validation/CV pipeline. This
    is generic β€” it inspects dtypes only, makes no dataset-specific assumption, and leaves numeric,
    datetime, and already-object columns untouched."""
    for col in df.columns:
        dt = df[col].dtype
        try:
            is_str_ext = isinstance(dt, pd.StringDtype) or (
                pd.api.types.is_extension_array_dtype(dt) and pd.api.types.is_string_dtype(dt)
            )
        except Exception:
            is_str_ext = False
        if is_str_ext:
            df[col] = df[col].astype(object)
    return df


def read_csv_dataset(path: str | Path, nrows: int | None = None, *, coerce_numeric: bool = True) -> pd.DataFrame:
    """Read a CSV with light schema repair for exported biomedical tables.

    For files above ``_LARGE_FILE_BYTES`` and when the caller did not request an explicit ``nrows``, a
    memory-bounded REPRESENTATIVE (systematic, whole-file) row sample is read so a multi-GB upload
    cannot exhaust the instance while still covering all participants."""
    path = Path(path)
    if nrows is None:
        _cap = _bounded_read_rows(path)
        if _cap is not None:
            df = _read_csv_systematic_sample(path, _cap)
            df = df.loc[:, ~df.columns.astype(str).str.match(r"^Unnamed: 0$")]
            return _normalize_string_extension_dtypes(_coerce_numeric_like_columns(df) if coerce_numeric else df)
    df = _robust_read_csv(path, nrows=nrows, low_memory=False)
    unnamed_fraction = sum(str(column).startswith("Unnamed") for column in df.columns) / max(1, len(df.columns))
    if unnamed_fraction > 0.35:
        probe = _robust_read_csv(path, header=None, nrows=3)
        second_row = probe.iloc[1].dropna().astype(str).str.lower().tolist() if len(probe) > 1 else []
        header_like = any(token in " ".join(second_row) for token in ("participant", "homa", "label", "heart", "steps"))
        if header_like:
            df = _robust_read_csv(path, header=1, nrows=nrows, low_memory=False)
    df = df.loc[:, ~df.columns.astype(str).str.match(r"^Unnamed: 0$")]
    if coerce_numeric:
        df = _coerce_numeric_like_columns(df)
    return _normalize_string_extension_dtypes(df)


_ID_KEY_PRIORITY = (
    "participant_id",
    "subject_id",
    "user_id",
    "person_id",
    "patient_id",
    "pid",
    "id",
)


def _short_source_name(name: str) -> str:
    base = Path(str(name or "dataset")).stem
    safe = "".join(ch if ch.isalnum() else "_" for ch in base).strip("_").lower()
    return (safe or "dataset")[:24]


def _detect_join_key(frames: list[pd.DataFrame]) -> str | None:
    """Pick an ID-like column present in *every* frame, by priority then by name pattern."""
    common = set(frames[0].columns)
    for frame in frames[1:]:
        common &= set(frame.columns)
    if not common:
        return None
    lowered = {str(col).lower(): col for col in common}
    for key in _ID_KEY_PRIORITY:
        if key in lowered:
            return lowered[key]
    for low, original in lowered.items():
        if low.endswith("_id") or low == "id" or "participant" in low or "subject" in low:
            return original
    return None


def merge_datasets(named_frames: list[tuple[str, pd.DataFrame]]) -> tuple[pd.DataFrame, dict]:
    """Combine several uploaded CSVs into one analysis frame so the agent can reason across
    them jointly.

    Strategy:
      * **join** on a shared ID column (e.g. participant_id) when one exists in every file β€”
        the common case of "wearable features in one file, clinical labels in another".
      * **union** (stack rows) when the files share an identical schema but no ID key.
      * otherwise raise InsufficientDataError so the caller can explain the boundary rather
        than silently producing a meaningless cross-product.

    Colliding non-key column names are prefixed with the source's short name so provenance is
    preserved and nothing is silently overwritten.
    """
    named_frames = [(name, frame) for name, frame in named_frames if frame is not None and len(frame.columns)]
    if len(named_frames) < 2:
        raise InsufficientDataError("merge_datasets needs at least two datasets.")
    names = [_short_source_name(name) for name, _ in named_frames]
    # Disambiguate duplicate source names so prefixes stay unique.
    seen: dict[str, int] = {}
    for index, name in enumerate(names):
        if name in seen:
            seen[name] += 1
            names[index] = f"{name}_{seen[name]}"
        else:
            seen[name] = 0
    frames = [frame for _, frame in named_frames]
    sources = [
        {"name": names[i], "rows": int(len(frames[i])), "columns": int(frames[i].shape[1])}
        for i in range(len(frames))
    ]

    key = _detect_join_key(frames)
    if key is not None:
        # Prefix only the columns that collide across files (keep clean names otherwise).
        non_key_counts: dict[str, int] = {}
        for frame in frames:
            for col in frame.columns:
                if col != key:
                    non_key_counts[col] = non_key_counts.get(col, 0) + 1
        colliding = {col for col, count in non_key_counts.items() if count > 1}
        renamed: dict[str, str] = {}
        prepared = []
        for source_name, frame in zip(names, frames):
            frame = frame.copy()
            frame[key] = frame[key].astype(str)
            rename_map = {}
            for col in frame.columns:
                if col != key and col in colliding:
                    new_col = f"{source_name}_{col}"
                    rename_map[col] = new_col
                    renamed[f"{source_name}.{col}"] = new_col
            if rename_map:
                frame = frame.rename(columns=rename_map)
            prepared.append(frame)
        # Cardinality / row-explosion detection: a one-to-many or many-to-many join REPLICATES
        # rows from the smaller file(s), so the merged rows are NOT independent observations.
        # Surfacing this prevents silent pseudo-replication (inflated n / significance).
        per_source_card = []
        sources_with_dupe_keys = 0
        for src_name, frame in zip(names, prepared):
            vc = frame[key].value_counts()
            max_per_key = int(vc.max()) if len(vc) else 0
            per_source_card.append({
                "name": src_name, "rows": int(len(frame)), "unique_keys": int(frame[key].nunique()),
                "max_rows_per_key": max_per_key, "duplicate_key_rate": round(float((vc > 1).mean()) if len(vc) else 0.0, 3),
            })
            if max_per_key > 1:
                sources_with_dupe_keys += 1
        merged = prepared[0]
        for frame in prepared[1:]:
            merged = merged.merge(frame, on=key, how="outer")
        rows_before = sum(int(len(f)) for f in prepared)
        largest = max((int(len(f)) for f in prepared), default=0)
        rows_after = int(len(merged))
        cardinality = "many_to_many" if sources_with_dupe_keys >= 2 else ("one_to_many" if sources_with_dupe_keys == 1 else "one_to_one")
        key_sets = [set(f[key].dropna().unique()) for f in prepared]
        common = set.intersection(*key_sets) if key_sets else set()
        all_keys = set().union(*key_sets) if key_sets else set()
        unmatched_keys = int(len(all_keys) - len(common))
        explosion_warning = None
        if cardinality != "one_to_one":
            explosion_warning = (
                f"{cardinality.replace('_', '-')} join on '{key}': rows from the smaller file(s) are REPLICATED "
                f"across matches (merged to {rows_after:,} rows from sources of "
                f"{', '.join(str(c['rows']) for c in per_source_card)}). These {rows_after:,} rows are NOT "
                f"independent observations β€” treat '{key}' as a participant/cluster id (mixed-effects or "
                f"participant-level split), otherwise per-row significance is inflated."
            )
        info = {
            "strategy": "join",
            "join_key": key,
            "sources": sources,
            "rows": rows_after,
            "rows_before_join": rows_before,
            "columns": int(merged.shape[1]),
            "renamed_columns": renamed,
            "cardinality": cardinality,
            "row_multiplier": round(rows_after / largest, 2) if largest else None,
            "per_source_key_cardinality": per_source_card,
            "unmatched_keys": unmatched_keys,
            "explosion_warning": explosion_warning,
        }
        return merged, info

    # No shared key: only safe to union if every file has the same column set.
    column_sets = [tuple(frame.columns) for frame in frames]
    if len(set(column_sets)) == 1:
        stacked = pd.concat(
            [frame.assign(__source=name) for name, frame in zip(names, frames)],
            ignore_index=True,
        )
        info = {
            "strategy": "union",
            "join_key": None,
            "sources": sources,
            "rows": int(len(stacked)),
            "columns": int(stacked.shape[1]),
            "renamed_columns": {},
        }
        return stacked, info

    raise InsufficientDataError(
        "These datasets don't share an ID column (e.g. participant_id) to join on, and their "
        "columns differ, so they can't be combined automatically. Give them a common ID column, "
        "or analyze one dataset at a time."
    )


def _coerce_numeric_like_columns(df: pd.DataFrame) -> pd.DataFrame:
    """Promote numeric-looking object columns without touching datetime-like text.

    User uploads often contain commas, percent signs, or currency-like prefixes that
    make pandas treat otherwise numeric sensor/lab columns as strings. CoDaS should
    still audit and model those columns, while preserving dates and free text.
    """
    work = df.copy()
    for column in work.columns:
        series = work[column]
        if not pd.api.types.is_object_dtype(series) and not pd.api.types.is_string_dtype(series):
            continue
        if _looks_like_datetime(series):
            continue
        cleaned = (
            series.astype("string")
            .str.replace(",", "", regex=False)
            .str.replace("%", "", regex=False)
            .str.replace("$", "", regex=False)
            .str.strip()
        )
        observed = cleaned.notna() & (cleaned != "")
        if int(observed.sum()) < 20:
            continue
        numeric = pd.to_numeric(cleaned, errors="coerce")
        if float(numeric.notna().sum() / max(1, observed.sum())) >= 0.8:
            work[column] = numeric
    return work


def _looks_like_datetime(series: pd.Series) -> bool:
    if pd.api.types.is_datetime64_any_dtype(series):
        return True
    if not pd.api.types.is_object_dtype(series):
        return False
    sample = series.dropna().astype(str).head(30)
    if sample.empty:
        return False
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", UserWarning)
        parsed = pd.to_datetime(sample, errors="coerce")
    return parsed.notna().mean() >= 0.8


def _name_tokens(name: str) -> set[str]:
    """Lowercased alphanumeric tokens of a column name, split on any non-alnum boundary.
    'f_slp:..._rapids_...' -> {'f','slp',...,'rapids',...}; 'participant_id' -> {'participant','id'}."""
    norm = "".join(ch.lower() if ch.isalnum() else " " for ch in str(name))
    return {t for t in norm.split() if t}


def _hinted(columns: list[str], hints: tuple[str, ...]) -> list[str]:
    """Columns whose NAME TOKENS contain a hint. Token-based (not substring) so an identifier
    hint like 'pid' matches a 'pid' / 'participant_pid' column but NOT 'f_..._rapids_...' β€” the
    GLOBEM/RAPIDS feature family, where 'rapids' embeds the substring 'pid'. A multi-token hint
    ('record_id', 'user_id') matches only when ALL its tokens are present. (Substring matching
    here false-matched 3,591 RAPIDS features and collapsed 8,225 rows into 2 'participants'.)"""
    hint_tokens = [tuple(_name_tokens(h)) for h in hints]
    out: list[str] = []
    for column in columns:
        toks = _name_tokens(column)
        if any(ht and set(ht).issubset(toks) for ht in hint_tokens):
            out.append(column)
    return out


def _detect_participant_column(df: pd.DataFrame, target_column: str | None = None) -> str | None:
    """Detect the participant / cluster identifier from column names + structure.

    Repeated-measures rows from the same person are NOT independent: treating them as
    independent inflates effective n and significance (pseudo-replication). The participant
    column drives aggregation/mixed-effects, so missing it is a high-impact correctness bug.
    Name detection alone is fragile -- clinical & standard identifiers (record_id, patient_id,
    usubjid, mrn, bare 'id') are not in the original hint list -- so we (1) match an expanded
    clinical/standard hint list, preferring a column that actually clusters (repeats), then
    (2) fall back to any identifier-NAMED column (id / *_id / *code / *key) that clusters.
    Detection stays high-precision (identifier-named only) to avoid mis-aggregating a real
    low-cardinality feature (hour, site, num_visits) as if it were the subject id.
    """
    n = int(len(df))
    if n == 0:
        return None
    cols = [c for c in df.columns if c != target_column]

    def k_of(column: str) -> int:
        try:
            return int(df[column].nunique(dropna=True))
        except Exception:
            return n

    # (1) Name-hinted identifiers (expanded). Prefer one that genuinely clusters rows; among
    # several, the participant/cluster unit is the one with the FEWEST groups (most rows each),
    # so a near-unique sample_id never wins over a repeating subject id. BUT a canonical SUBJECT
    # identifier (participant_id / subject_id / *_id) must outrank a generic cluster/category hint
    # (group / cluster / household / unit_id) β€” otherwise a low-cardinality demographic CATEGORY
    # (e.g. a 3-value 'household' = lives-alone/with-others) wrongly beats a real 'participant_id'
    # by having the fewest groups, mis-aggregating millions of rows into a handful of "participants".
    def _id_priority(column: str) -> int:
        toks = _name_tokens(column)  # token-based (not substring): 'pid' must not score 'rapids'
        if toks & {"participant", "subject", "subjid", "usubjid", "wearer"}:
            return 0  # canonical per-person identifier
        if (toks & {"person", "patient", "pid", "mrn", "studyid"}
                or {"user", "id"} <= toks or {"record", "id"} <= toks or {"study", "id"} <= toks):
            return 1  # other per-person identifier
        if "id" in toks:
            return 2  # generic *_id (could be a record/sample id)
        return 3      # generic cluster/category hint: group, cluster, household, unit_id

    hinted = _hinted(cols, ID_HINTS)
    if hinted:
        clustering = [c for c in hinted if 1 < k_of(c) < n]
        pool = clustering or hinted
        best_pri = min(_id_priority(c) for c in pool)
        tier = [c for c in pool if _id_priority(c) == best_pri]
        # within the strongest name tier, the FEWEST-groups column is the subject/cluster unit
        return min(tier, key=k_of)

    # (2) Structural fallback: an identifier-NAMED column that clusters (>= 2 rows per id).
    structural = []
    for c in cols:
        name = str(c).strip().lower()
        idish = (
            name in ("id", "uid", "uuid", "guid")
            or name.endswith("_id")
            or name.endswith("code")
            or name.endswith("key")
        )
        if not idish:
            continue
        k = k_of(c)
        if 1 < k < n and (n / k) >= 2:
            structural.append(c)
    if structural:
        return min(structural, key=k_of)
    return None


# Demographic / SES tokens β€” these are CONFOUNDERS or keys, never wearable biomarker CANDIDATES
# (e.g. has_disability, education, income, marital_status). They may still be adjusted for as
# confounders, but must not be promoted as "discovered biomarkers". Token-based (split on non-alnum)
# so "age" can't hit "usage"/"average" and "race" can't hit "embrace".
_DEMOGRAPHIC_FEATURE_TOKENS = frozenset({
    "age", "gender", "sex", "race", "ethnicity", "ethnic", "hispanic", "latinx", "marital",
    "education", "educational", "income", "financial", "occupation", "employment", "disability",
    "disabled", "household", "demographic", "demographics", "socioeconomic", "ses",
    # marital-status family: a real dwb run surfaced `is_married` (ρ=-0.33) as an "exploratory"
    # candidate because only "marital" was a token. Marital status is a SOCIAL confounder of
    # depression, not a wearable biomarker. High-precision tokens only (NOT "single", which would
    # false-match wearable features like single_bout / single_use).
    "married", "unmarried", "marriage", "spouse", "spousal", "divorced", "widowed", "widow",
})


def _is_demographic_column(column: str) -> bool:
    """True for demographic/SES columns that must not be promoted as biomarker candidates."""
    norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
    toks = {t for t in norm.split("_") if t}
    return bool(toks & _DEMOGRAPHIC_FEATURE_TOKENS)


# Outcome / label / clinical-endpoint tokens β€” these are OUTCOMES, never biomarker CANDIDATES.
# Using one outcome measure to "predict" another operationalization of the same construct is
# construct circularity / label leakage. On GLOBEM the chosen target is `target_depression_weekly`
# while `target_BDI2`, `target_phq4`, `target_dep`, `target_depression_merged`, `target_anxiety_weekly`
# are ALTERNATE depression/anxiety labels β€” predicting depression from another depression scale is a
# credibility disaster ("depression predicts depression"). Such columns can be the TARGET or a
# covariate, but must never surface as a discovered "biomarker". HIGH-PRECISION token set: explicit
# label markers + validated psychiatric/clinical instrument acronyms ONLY β€” deliberately NOT broad
# tokens like "score"/"severity"/"glucose"/"insulin" that are legitimate sensor biomarkers elsewhere.
_OUTCOME_LABEL_TOKENS = frozenset({
    "target", "targets", "outcome", "outcomes", "label", "labels", "groundtruth", "endpoint",
    "diagnosis", "diagnosed", "diagnostic",
    "phq", "phq2", "phq4", "phq8", "phq9", "gad", "gad2", "gad7", "bdi", "bdi2", "cesd", "cesd10",
    "pss", "panas", "stai", "ders", "ucla", "psqi", "who5",
})


def _is_outcome_label_column(column: str) -> bool:
    """True for outcome/label/clinical-endpoint columns that must not be biomarker candidates
    (they may be the target or a covariate, but predicting one outcome from another is circular).
    Token-based + aggregation-suffix robust ('target_BDI2_mean' -> {target,bdi2,mean} matches)."""
    norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
    toks = {t for t in norm.split("_") if t}
    return bool(toks & _OUTCOME_LABEL_TOKENS)


# Psychometric / self-report target tokens. When the TARGET is one of these (or any "*_score"
# self-report instrument), sibling "*_score" SURVEY columns are CONSTRUCT OVERLAP β€” self-report
# predicting self-report, not a wearable biomarker. On dwb, predicting phq_score (PHQ-9, which itself
# contains sleep & fatigue items) from sleep_disturbance_score / tiredness_score / sleep_impairment_score
# is criterion contamination ("depression symptoms predict depression"). The agent layer
# (orchestration.infer_exclusions) already strips these for the UI; mirroring it here makes the CORE
# ENGINE self-protecting so a direct run_discovery / API / harness call gets the same guard.
_PSYCHOMETRIC_TARGET_TOKENS = frozenset({
    "phq", "gad", "pss", "depress", "depression", "anxiety", "anxious", "mood", "stress",
    "wellbe", "wellbeing", "affect", "psych", "panas", "who5", "cesd", "bdi", "stai", "ders",
})
# Wearable / device-derived scores that ARE legitimate candidates even against a psychometric target.
_WEARABLE_SCORE_ALLOWLIST = (
    "step", "heart_rate", "resting_heart", "hrv", "unlock", "screen", "accel", "bins",
    "min_in_", "distance", "calorie", "wear", "sleep_efficiency", "activity", "cadence",
)


def _target_is_psychometric(target_column: str | None) -> bool:
    if not target_column:
        return False
    norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(target_column)).strip("_")
    toks = {t for t in norm.split("_") if t}
    return norm.endswith("_score") or norm == "score" or bool(toks & _PSYCHOMETRIC_TARGET_TOKENS)


def _is_construct_overlap_score(column: str, target_column: str | None) -> bool:
    """A self-report '*_score' SURVEY sibling of a psychometric/self-report target is construct
    overlap (self-report predicting self-report), not a wearable biomarker. Wearable device scores
    (sleep_efficiency, step, heart_rate, ...) are allow-listed. Operates on RAW column names (run
    in build_analysis_frame BEFORE longitudinal aggregation, so 'sleep_disturbance_score' is caught
    before it becomes 'sleep_disturbance_score_mean')."""
    if not target_column or column == target_column:
        return False
    norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
    if not (norm.endswith("_score") or norm == "score"):
        return False
    if any(tok in norm for tok in _WEARABLE_SCORE_ALLOWLIST):
        return False
    return _target_is_psychometric(target_column)


def _suggested_target_columns(columns: list[str], numeric_columns: list[str]) -> list[str]:
    """Rank plausible endpoints ahead of generic demographic survey scores."""
    exact_priority = {
        "depression_status": 130,
        "depression_binary": 125,
        "madrs_start": 120,
        "madrs_end": 115,
        "madrs_change": 110,
        "true_homa_ir": 100,
        "homa_ir": 100,
        "depression_score": 100,
        "phq_score": 100,
        "phq_total": 100,
        "insulin_resistance_score": 100,
        "relapse_label": 100,
        "fatigue_score": 100,
        "outcome_severity_score": 100,
    }
    domain_tokens = (
        "homa",
        "insulin",
        "glucose",
        "hba1c",
        "phq",
        "madrs",
        "depression",
        "anxiety",
        "gad",
        "relapse",
        "fatigue",
        "outcome",
        "target",
        "label",
        "severity",
        # psychometric / clinical endpoints common in wearable + EMA studies
        "pss",
        "stress",
        "mood",
        "wellbeing",
        "mortality",
        "death",
        "diagnosis",
        "disease",
        "dementia",
        "updrs",
    )
    # Demographic / SES / identifier tokens are NEVER a discovery endpoint (they are confounders or
    # keys). Token-based exclusion so "age" doesn't match "usage" and "race" doesn't match "embrace".
    nonoutcome_tokens = {
        "age", "gender", "sex", "race", "ethnicity", "ethnic", "hispanic", "latinx", "marital",
        "education", "income", "financial", "occupation", "employment", "disability", "household",
        "id", "participant", "subject", "user", "pid", "mrn", "uuid",
        # Provenance, study-design, and sensor-placement descriptors can mention a disease or
        # endpoint domain without being an analyzable outcome.
        "context", "source", "dataset", "study", "cohort", "site", "center", "protocol",
        "placement", "placements", "sensor", "sensors", "annotation", "annotations",
        "provenance", "metadata", "device", "firmware",
    }
    generic_penalties = (
        "age_score",
        "gender_score",
        "sex_score",
        "race",
        "ethnicity",
        "financial",
        "income",
        "bfi",
        "intake_survey",
        "duration_score",
        "composition_score",
    )
    scored: list[tuple[int, int, str]] = []
    for column in columns:
        normalized = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
        toks = {t for t in normalized.split("_") if t}
        # Exclude demographics/SES/ids and any time axis from "likely targets" β€” these horrified the
        # user when surfaced (e.g. "age" as a likely target on a 4,710-participant wearable file).
        if toks & nonoutcome_tokens or _likely_time_column_name(column):
            continue
        score = 0
        if normalized in exact_priority:
            score += exact_priority[normalized]
        score += sum(12 for token in domain_tokens if token in normalized)
        if any(token in normalized for token in generic_penalties):
            score -= 30
        # NOTE: no blanket "+3 because numeric". A suggested target must carry a real endpoint signal
        # (exact-priority name or a clinical/psychometric token); otherwise sensor features (steps,
        # sleep minutes) and survey items masquerade as outcomes. profile_dataframe falls back to
        # numeric_columns[:5] only when NO endpoint-signal column exists, so token-less labels still surface.
        if score > 0:
            scored.append((score, -len(str(column)), column))
    scored.sort(reverse=True)
    return [column for _, _, column in scored]


def _likely_time_column_name(column: str) -> bool:
    """Return true for actual time axes, not summary features like weekly means."""
    name = str(column).lower()
    normalized = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
    summary_tokens = ("mean", "median", "std", "min", "max", "weekly", "hourly", "7day")
    if any(token in normalized.split("_") for token in summary_tokens):
        return False
    exact = {"date", "day", "hour", "week", "time", "timestamp", "datetime", "study_week"}
    if normalized in exact:
        return True
    suffixes = ("_date", "_day", "_hour", "_week", "_time", "_timestamp", "_datetime")
    prefixes = ("date_", "day_", "hour_", "week_", "time_", "timestamp_", "datetime_")
    if normalized.endswith(suffixes) or normalized.startswith(prefixes):
        return True
    return bool(
        any(token in name for token in ("datetime", "timestamp"))
        or ("date" in name and not any(token in name for token in ("duration", "update")))
    )


def profile_dataframe(
    df: pd.DataFrame,
    target_column: str | None = None,
    participant_id_column: str | None = None,
    time_column: str | None = None,
) -> DatasetProfile:
    numeric_columns = [column for column in df.columns if pd.api.types.is_numeric_dtype(df[column])]
    datetime_columns = [column for column in df.columns if _looks_like_datetime(df[column])]
    categorical_columns = [
        column
        for column in df.columns
        if column not in numeric_columns and column not in datetime_columns
    ]
    suggested_targets = _suggested_target_columns(list(df.columns), numeric_columns)

    if participant_id_column is None:
        participant_id_column = _detect_participant_column(df, target_column)
    if time_column is None:
        time_candidates = [column for column in df.columns if _likely_time_column_name(str(column))]
        time_column = time_candidates[0] if time_candidates else None

    missing_fraction = {
        str(column): float(df[column].isna().mean())
        for column in df.columns
    }
    return DatasetProfile(
        rows=int(len(df)),
        columns=int(len(df.columns)),
        numeric_columns=numeric_columns,
        categorical_columns=categorical_columns,
        datetime_columns=datetime_columns,
        missing_fraction=missing_fraction,
        suggested_targets=suggested_targets,
        target_column=target_column,
        participant_id_column=participant_id_column,
        time_column=time_column,
    )


def _add_temporal_features(
    work: pd.DataFrame,
    numeric: list[str],
    participant_id_column: str,
    time_column: str,
    out_index: pd.Index,
) -> pd.DataFrame | None:
    """Per-participant temporal features that summary statistics cannot capture:

      * `<feat>_trendslope`          β€” OLS slope of the feature vs time (within-person trend; S16).
      * `<feat>_weekend_minus_weekday` β€” weekend-mean minus weekday-mean (social-jetlag style; S15).
      * `<feat>_night_over_day`      β€” night (00-06h) over day mean ratio (circadian; S14), only when
                                       the timestamp has sub-daily resolution.

    All are computed vectorised per feature (no per-group Python loop) and bounded by a budget so a
    very wide/long file degrades gracefully to summary stats only.
    """
    parsed = pd.to_datetime(work[time_column], errors="coerce")
    if parsed.notna().mean() < 0.6:
        return None
    n_participants = work[participant_id_column].nunique(dropna=True)
    # Budget: skip the enhancement on very large feature*participant grids (it is additive, not core).
    if not numeric or n_participants < 2 or (len(numeric) * max(1, n_participants)) > 6_000_000:
        return None

    tmp = work[[participant_id_column] + numeric].copy()
    # Continuous time in days-from-first (works for daily and hourly cadence alike).
    t0 = parsed.min()
    tmp["_t"] = (parsed - t0) / pd.Timedelta(days=1)
    tmp["_hour"] = parsed.dt.hour
    tmp["_dow"] = parsed.dt.dayofweek  # 0=Mon .. 6=Sun
    gid = tmp[participant_id_column]

    new_cols: dict[str, pd.Series] = {}

    # (1) Trend slope: cov(t,x)/var(t) within participant β€” vectorised via centered sums.
    t_centered = tmp["_t"] - gid.map(tmp.groupby(participant_id_column)["_t"].mean())
    t_var = (t_centered ** 2).groupby(gid).sum()
    valid_slope = t_var > 0
    for col in numeric:
        x_centered = tmp[col] - gid.map(tmp.groupby(participant_id_column)[col].mean())
        num = (t_centered * x_centered).groupby(gid).sum()
        slope = (num / t_var.where(valid_slope)).reindex(out_index)
        new_cols[f"{col}_trendslope"] = slope

    # (2) Weekday vs weekend contrast (needs both present across the dataset).
    has_weekend = bool((tmp["_dow"] >= 5).any() and (tmp["_dow"] < 5).any())
    if has_weekend:
        wk_mask = tmp["_dow"] < 5
        for col in numeric:
            wk = tmp.loc[wk_mask].groupby(participant_id_column)[col].mean()
            we = tmp.loc[~wk_mask].groupby(participant_id_column)[col].mean()
            new_cols[f"{col}_weekend_minus_weekday"] = (we - wk).reindex(out_index)

    # (3) Circadian night/day ratio (needs sub-daily resolution: >3 distinct hours).
    has_subdaily = int(tmp["_hour"].nunique(dropna=True)) > 3
    if has_subdaily:
        night_mask = tmp["_hour"] < 6
        for col in numeric:
            night = tmp.loc[night_mask].groupby(participant_id_column)[col].mean()
            day = tmp.loc[~night_mask].groupby(participant_id_column)[col].mean()
            new_cols[f"{col}_night_over_day"] = (night / day.replace(0, np.nan)).reindex(out_index)

    if not new_cols:
        return None
    out = pd.DataFrame(new_cols, index=out_index)
    # Drop columns that are entirely NaN or constant (no information / would just be noise).
    keep = [c for c in out.columns if out[c].notna().sum() >= 3 and out[c].nunique(dropna=True) > 1]
    return out[keep] if keep else None


def _aggregate_longitudinal(
    df: pd.DataFrame,
    target_column: str,
    participant_id_column: str,
    time_column: str | None,
    excluded_columns: set[str],
    confounder_columns: list[str] | None = None,
) -> pd.DataFrame:
    reserved = {target_column, participant_id_column}
    if time_column:
        reserved.add(time_column)
    confounders = [column for column in (confounder_columns or []) if column in df.columns]
    reserved.update(confounders)
    work = df.copy()
    for column in list(work.columns):
        if column in reserved or column in excluded_columns:
            continue
        clock_feature = _clock_hour_feature(work[column], str(column))
        if clock_feature is not None:
            work[f"{column}_clock_hour"] = clock_feature

    numeric = [
        column
        for column in work.columns
        if column not in reserved
        and column not in excluded_columns
        and not _is_demographic_column(column)  # demographics are confounders, not biomarker candidates
        and not _is_outcome_label_column(column)  # other outcome/label columns are circular, not candidates
        and pd.api.types.is_numeric_dtype(work[column])
    ]
    if not numeric:
        raise ValueError("No numeric sensor columns are available for longitudinal aggregation.")

    aggregations = {column: ["mean", "std", "min", "max", "median"] for column in numeric}
    feature_df = work.groupby(participant_id_column).agg(aggregations)
    feature_df.columns = [f"{name}_{agg}" for name, agg in feature_df.columns]
    # Build all coefficient-of-variation columns in ONE concat. Inserting them one-by-one
    # (feature_df[cv]=...) fragments the already-wide aggregate frame β€” O(cols^2) and a flood of
    # pandas PerformanceWarnings on a 100+-column wearable file, which measurably slows feature
    # engineering. Values are identical; only the assembly is vectorised.
    cv_data = {
        f"{column}_cv": feature_df[f"{column}_std"] / feature_df[f"{column}_mean"].replace(0, np.nan).abs()
        for column in numeric
    }
    cv_columns = list(cv_data.keys())
    if cv_data:
        feature_df = pd.concat([feature_df, pd.DataFrame(cv_data, index=feature_df.index)], axis=1)

    # --- Temporal feature engineering (S14/S15/S16) ---
    # Summary stats (mean/std/min/max/median/cv) alone miss biomarkers whose signal lives in the
    # TIME STRUCTURE: a within-person trend/slope, a weekday-vs-weekend contrast, or a circadian
    # (time-of-day) pattern. When a parseable time column exists, engineer per-participant temporal
    # features so these signals enter screening instead of being invisible. Bounded by a budget and
    # wrapped in try/except so it can never break a discovery (degrades to summary stats only).
    if time_column and time_column in work.columns:
        try:
            # Guard: skip temporal feature engineering for columns that look like outcome
            # proxies or label derivatives. Without this guard, a near-perfect outcome proxy
            # (e.g. daily_mood_selfreport, Οβ‰ˆ0.999 with the target) would be blocked by the
            # construct-validity gate in raw form β€” but its temporal derivative (trendslope,
            # Οβ‰ˆ0.26) would slip through the same gate and get validated, which is circular.
            # Strategy: also exclude columns that are strongly correlated with the target
            # in the aggregate frame (|rho| >= 0.95 at the participant level).
            _temporal_numeric = [col for col in numeric if col not in excluded_columns]
            if _temporal_numeric and target_column in df.columns:
                try:
                    # Block temporal feature engineering for construct-circular raw features.
                    # Compute participant-level aggregate correlation with the target for each
                    # candidate raw feature and skip those with |rho| >= 0.95 (near-identical
                    # to the target β†’ any temporal derivative is equally circular).
                    _target_agg = df.groupby(participant_id_column)[target_column].median()
                    from .statistics import safe_spearman as _ss
                    _construct_blocked: set[str] = set()
                    for _col in list(_temporal_numeric):
                        if _col not in df.columns:
                            continue
                        _col_agg = pd.to_numeric(df[_col], errors="coerce")
                        if _col_agg.notna().sum() < 10:
                            continue
                        _col_part = df.groupby(participant_id_column)[_col].mean()
                        _r, _, _ = _ss(_col_part.reindex(_target_agg.index), _target_agg)
                        import numpy as _np2
                        if _np2.isfinite(_r) and abs(_r) >= 0.95:
                            _construct_blocked.add(_col)
                    if _construct_blocked:
                        _temporal_numeric = [col for col in _temporal_numeric if col not in _construct_blocked]
                except Exception:
                    pass  # guard is best-effort; never fail the core aggregation
            _temporal = _add_temporal_features(
                work, _temporal_numeric, participant_id_column, time_column, feature_df.index,
            )
            if _temporal is not None and not _temporal.empty:
                feature_df = pd.concat([feature_df, _temporal], axis=1)
        except Exception:
            pass  # temporal features are an enhancement; never fail the core aggregation

    target_values = df[target_column].dropna()
    if target_values.nunique(dropna=True) == 2:
        target_df = df.groupby(participant_id_column)[target_column].max().rename(target_column)
        audit_target_note = "max/event aggregation for binary target"
    else:
        target_df = df.groupby(participant_id_column)[target_column].median().rename(target_column)
        audit_target_note = "median aggregation for continuous target"
    confounder_df = None
    if confounders:
        confounder_parts = []
        grouped = work.groupby(participant_id_column)
        for column in confounders:
            if pd.api.types.is_numeric_dtype(df[column]):
                confounder_parts.append(grouped[column].median().rename(column))
            else:
                confounder_parts.append(grouped[column].agg(lambda values: values.dropna().iloc[0] if values.dropna().size else np.nan).rename(column))
        confounder_df = pd.concat(confounder_parts, axis=1) if confounder_parts else None
    merged = feature_df.join(target_df)
    if confounder_df is not None:
        merged = merged.join(confounder_df)
    merged = merged.reset_index()
    merged.attrs["codas_target_aggregation_note"] = audit_target_note
    return merged


def _clock_hour_feature(series: pd.Series, column_name: str) -> pd.Series | None:
    """Extract a time-of-day hour feature from sleep/wake timestamp columns."""
    name = column_name.lower()
    if not any(token in name for token in ("sleep", "bed", "wake", "start", "end", "left", "entered")):
        return None
    if not any(token in name for token in ("time", "datetime", "timestamp")):
        return None
    if pd.api.types.is_numeric_dtype(series):
        return None
    sample = series.dropna().astype(str).head(40)
    if sample.empty:
        return None
    # Suppress pandas' per-call "Could not infer format" UserWarning. On ULTRA-WIDE files
    # (GLOBEM: 5,529 cols, many object "..._sleep_..._starttime_..." columns pass the name gate)
    # this function is called per column, and emitting+formatting thousands of identical warnings to
    # stderr is itself an I/O bottleneck that stalled discovery (>9 min). Parsing stays correct.
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", UserWarning)
        parsed = pd.to_datetime(sample, errors="coerce")
        if parsed.notna().mean() < 0.65:
            return None
        parsed_all = pd.to_datetime(series.astype("string"), errors="coerce")
    return parsed_all.dt.hour + parsed_all.dt.minute / 60.0 + parsed_all.dt.second / 3600.0


def _feature_match_score(column: str, include: tuple[str, ...], exclude: tuple[str, ...] = ()) -> int:
    # Short clinical tokens (<=3 chars: ast, alt, hdl, hrv, rhr) must match a WHOLE word-part, not an
    # arbitrary substring β€” otherwise 'ast'/'alt' match the fetal-cardiotocography columns 'astv'/'altv'
    # (abnormal short-/long-term variability) and CoDaS fabricates a nonsensical liver De-Ritis AST/ALT
    # ratio from fetal heart-rate features. Longer tokens keep substring matching (e.g. 'chol' in
    # 'cholesterol', 'albumin' in 'serum_albumin'), so legitimate detections (incl. ILPD sgot_ast/sgpt_alt)
    # are unchanged.
    normalized = column.lower()
    spaced = " " + re.sub(r"[^a-z0-9]+", " ", normalized).strip() + " "
    def _hit(token: str) -> bool:
        t = token.lower()
        if len(t) <= 3:
            return (" " + t + " ") in spaced
        return t in normalized
    if any(_hit(token) for token in exclude):
        return 0
    return sum(1 for token in include if _hit(token))


def _best_feature(columns: list[str], include: tuple[str, ...], exclude: tuple[str, ...] = ()) -> str | None:
    scored = [
        (_feature_match_score(column, include, exclude), -len(str(column)), column)
        for column in columns
    ]
    scored = [item for item in scored if item[0] > 0]
    if not scored:
        return None
    scored.sort(reverse=True)
    return scored[0][2]


def _add_named_ratio(
    frame: pd.DataFrame,
    numerator: str | None,
    denominator: str | None,
    ratio_name: str,
    feature_components: dict[str, list[str]],
    added: list[str],
) -> None:
    if not numerator or not denominator or numerator == denominator or ratio_name in frame.columns:
        return
    if numerator not in frame.columns or denominator not in frame.columns:
        return
    den = pd.to_numeric(frame[denominator], errors="coerce").replace(0, np.nan)
    num = pd.to_numeric(frame[numerator], errors="coerce")
    values = num / den
    if values.notna().sum() < 20 or values.nunique(dropna=True) <= 1:
        return
    frame[ratio_name] = values
    feature_components[ratio_name] = [numerator, denominator]
    added.append(ratio_name)


def _add_domain_ratio_features(
    frame: pd.DataFrame,
    feature_columns: list[str],
    feature_components: dict[str, list[str]],
    max_ratio_features: int,
) -> list[str]:
    """Add paper-aligned candidate families before generic ratio enumeration."""
    added: list[str] = []
    columns = list(feature_columns)
    lower_to_original = {str(column).lower(): column for column in columns}
    exact = lambda name: lower_to_original.get(name.lower())

    steps = _best_feature(columns, ("steps",), ("std", "cv", "min", "max"))
    rhr = _best_feature(columns, ("resting heart rate",), ("std", "cv", "min", "max")) or _best_feature(columns, ("rhr",), ("std", "cv", "min", "max"))
    hrv = _best_feature(columns, ("hrv",), ("std", "cv", "min", "max")) or _best_feature(columns, ("rmssd",), ("std", "cv", "min", "max"))
    ast = exact("ast") or _best_feature(columns, ("ast",), ())
    alt = exact("alt") or _best_feature(columns, ("alt",), ())
    albumin = _best_feature(columns, ("albumin",), ())
    globulin = _best_feature(columns, ("globulin",), ())
    cholesterol = _best_feature(columns, ("cholesterol",), ("hdl", "ldl")) or _best_feature(columns, ("chol",), ("hdl", "ldl"))
    hdl = exact("hdl") or _best_feature(columns, ("hdl",), ())

    proposals = [
        (steps, rhr, "cardiovascular_fitness_index_steps_over_resting_heart_rate"),
        (ast, alt, "derived_ast_over_alt_ratio_de_ritis"),
        (albumin, globulin, "derived_albumin_over_globulin_ratio"),
        (cholesterol, hdl, "derived_cholesterol_over_hdl_ratio"),
        (hrv, rhr, "derived_hrv_over_resting_heart_rate_ratio"),
    ]
    for numerator, denominator, name in proposals:
        if len(added) >= max_ratio_features:
            break
        _add_named_ratio(frame, numerator, denominator, name, feature_components, added)
    return added


def _add_ratio_features(
    frame: pd.DataFrame,
    feature_columns: list[str],
    feature_components: dict[str, list[str]],
    max_ratio_features: int,
) -> tuple[list[str], list[str]]:
    added: list[str] = []
    if max_ratio_features <= 0:
        return feature_columns, added

    domain_added = _add_domain_ratio_features(
        frame,
        feature_columns,
        feature_components,
        max_ratio_features=max_ratio_features,
    )
    added.extend(domain_added)
    remaining_slots = max(0, max_ratio_features - len(added))
    if remaining_slots <= 0:
        return feature_columns + added, added

    usable = []
    for column in feature_columns:
        values = pd.to_numeric(frame[column], errors="coerce")
        finite = values[np.isfinite(values)]
        if len(finite) < 20:
            continue
        if (finite.abs() > 1e-9).mean() < 0.95:
            continue
        usable.append(column)

    usable = usable[:12]
    for i, numerator in enumerate(usable):
        for denominator in usable[i + 1 :]:
            if len(added) >= max_ratio_features:
                return feature_columns + added, added
            ratio_name = f"{numerator}_over_{denominator}"
            reverse_name = f"{denominator}_over_{numerator}"
            if ratio_name in frame.columns or reverse_name in frame.columns:
                continue
            den = pd.to_numeric(frame[denominator], errors="coerce").replace(0, np.nan)
            rev_den = pd.to_numeric(frame[numerator], errors="coerce").replace(0, np.nan)
            frame[ratio_name] = pd.to_numeric(frame[numerator], errors="coerce") / den
            frame[reverse_name] = pd.to_numeric(frame[denominator], errors="coerce") / rev_den
            feature_components[ratio_name] = [numerator, denominator]
            feature_components[reverse_name] = [denominator, numerator]
            added.extend([ratio_name, reverse_name])
            if len(added) >= max_ratio_features:
                return feature_columns + added, added
    return feature_columns + added, added


def _add_interaction_features(
    frame: pd.DataFrame,
    feature_columns: list[str],
    feature_components: dict[str, list[str]],
    max_interaction_features: int = 12,
) -> list[str]:
    """Multiplicative INTERACTION features z(a)*z(b) for the leading numeric pairs (S5).

    Ratios capture one two-feature relationship; interactions capture another β€” an effect present only
    when two features are jointly high/low, with neither main effect alone. Standardized/centered so the
    product is a genuine interaction term (both-high & both-low β‡’ positive), not scale-dominated. Has its
    OWN budget so it is never crowded out by the ratio budget. Detects BETWEEN-unit interactions; pure
    within-person/within-row interactions still need a mixed-effects interaction term (documented).
    """
    if max_interaction_features <= 0:
        return []
    # Prioritise the PRIMARY signal carrier of each distinct base feature for pairing: the central-
    # tendency (`_mean`) aggregate, or the raw feature for cross-sectional data. Pairing the means of
    # DISTINCT base features (activity_index_mean Γ— sleep_index_mean) yields the interpretable, likely
    # interactions; pairing a feature with its own std/min/max is redundant. This makes the meaningful
    # interaction reachable within the small budget instead of being crowded out by same-family products.
    def _base_of(col: str) -> str:
        for suf in ("_mean", "_std", "_min", "_max", "_median", "_cv", "_trendslope",
                    "_weekend_minus_weekday", "_night_over_day"):
            if col.endswith(suf):
                return col[: -len(suf)]
        return col
    # Derive interaction candidates from the FULL frame, not the (already univariate-pruned)
    # feature_columns list: an interaction-only biomarker has null MAIN effects, so its components
    # are exactly the ones the univariate screen drops β€” they must still be reachable here.
    frame_cols = [str(c) for c in frame.columns]
    mean_cols = [c for c in frame_cols if c.endswith("_mean") and "_over_" not in c and "_x_" not in c]
    primary = mean_cols if mean_cols else [c for c in feature_columns if "_over_" not in c and "_x_" not in c]
    usable, seen_base = [], set()
    for column in primary:
        if "_over_" in column or "_x_" in column:  # skip already-derived ratio/interaction features
            continue
        base = _base_of(column)
        if base in seen_base:
            continue
        values = pd.to_numeric(frame[column], errors="coerce")
        finite = values[np.isfinite(values)]
        if len(finite) < 20 or finite.nunique() <= 2:
            continue
        usable.append(column)
        seen_base.add(base)
    usable = usable[:8]  # cap pair explosion: 8 distinct base features -> up to 28 pairs, budget-capped
    added: list[str] = []
    for i, a in enumerate(usable):
        for b in usable[i + 1:]:
            if len(added) >= max_interaction_features:
                return added
            inter_name = f"{a}_x_{b}"
            if inter_name in frame.columns:
                continue
            av = pd.to_numeric(frame[a], errors="coerce")
            bv = pd.to_numeric(frame[b], errors="coerce")
            sa = av.std(ddof=0)
            sb = bv.std(ddof=0)
            if not (sa and sb and np.isfinite(sa) and np.isfinite(sb)):
                continue
            prod = ((av - av.mean()) / sa) * ((bv - bv.mean()) / sb)
            if prod.notna().sum() < 20 or prod.nunique(dropna=True) <= 2:
                continue
            frame[inter_name] = prod
            feature_components[inter_name] = [a, b]
            added.append(inter_name)
    return added


def build_analysis_frame(
    df: pd.DataFrame,
    target_column: str,
    participant_id_column: str | None = None,
    time_column: str | None = None,
    excluded_columns: list[str] | None = None,
    confounder_columns: list[str] | None = None,
    max_ratio_features: int = 24,
) -> AnalysisFrame:
    if target_column not in df.columns:
        raise InsufficientDataError(f"Target column '{target_column}' is not present in the dataset.")
    # Constant target = nothing to associate/predict. (Too-few-observations is left
    # to the downstream screening gate, which fires AFTER participant aggregation so
    # legitimate small longitudinal cohorts are not blocked pre-aggregation.)
    _target_numeric = pd.to_numeric(df[target_column], errors="coerce").dropna()
    if len(_target_numeric) > 0 and float(_target_numeric.std(ddof=0) or 0.0) == 0.0:
        raise InsufficientDataError(
            f"Target '{target_column}' has no variance (all observed values are identical); there is nothing to associate or predict."
        )
    # Reject a target that is mostly non-numeric AND high-cardinality (a date,
    # identifier, or free-text column mistakenly chosen as the outcome). A binary
    # or low-cardinality categorical target is allowed and handled downstream.
    # NOTE: compute the numeric fraction over OBSERVED (non-null) values so a
    # heavily-missing but genuinely numeric target is not mistaken for text.
    _observed_target = df[target_column].dropna()
    _binary_target_note: str | None = None
    if len(_observed_target) > 0:
        _numeric_frac = float(pd.to_numeric(_observed_target, errors="coerce").notna().mean())
        _target_unique = int(_observed_target.nunique())
        if _numeric_frac < 0.5:
            # Non-numeric (text/categorical) target. A 2-class target is encoded to 0/1 so the
            # numeric screening/modeling can proceed (classification-style). High-cardinality looks
            # like a date/id/free-text column, and 3-10 unordered text categories (e.g. a multi-level
            # "sex" field) are not a usable single outcome β€” both are rejected with a clear message.
            # Without this, a low-cardinality text target slipped through and crashed the numeric
            # screening downstream (safe_spearman: "could not convert string to float: 'Male'").
            if _target_unique > 10:
                raise InsufficientDataError(
                    f"Target '{target_column}' is not a usable outcome: its observed values are mostly non-numeric with "
                    f"{_target_unique} distinct values, which looks like a date, identifier, or free-text column rather than a "
                    "clinical/behavioral endpoint. Choose a numeric or binary target."
                )
            if _target_unique == 2:
                _cats = list(pd.Categorical(_observed_target).categories)
                df = df.copy()
                _codes = pd.Series(pd.Categorical(df[target_column], categories=_cats).codes, index=df.index)
                df[target_column] = _codes.where(_codes >= 0)
                _binary_target_note = (
                    f"Encoded binary categorical target '{target_column}' as 0/1 ({_cats[0]}=0, {_cats[1]}=1)."
                )
            else:
                raise InsufficientDataError(
                    f"Target '{target_column}' is a non-numeric categorical column with {_target_unique} categories; "
                    "biomarker discovery requires a numeric or binary outcome. Choose a numeric endpoint, or a binary "
                    "target for classification."
                )
    excluded = set(excluded_columns or [])
    confounders = [column for column in (confounder_columns or []) if column in df.columns]
    audit_log: list[str] = []
    if _binary_target_note:
        audit_log.append(_binary_target_note)

    # Outcome/label columns OTHER than the target are construct-circular as biomarker candidates
    # (predicting one outcome from another). Block them up front and record the reason β€” this is the
    # NAME-based complement to the value-based construct-validity hard gate, which only catches
    # near-perfect proxies; a moderately-correlated alternative scale (e.g. GLOBEM target_BDI2 vs
    # target_depression_weekly, rho~0.6) would slip the value gate yet is still circular. Recorded in
    # excluded_columns + audit_log so the block is transparent (not a silent drop), and the column is
    # retained in the frame for reference rather than promoted as a "discovered biomarker".
    _blocked_outcomes = [
        column for column in df.columns
        if column != target_column and column not in confounders and column not in excluded
        and _is_outcome_label_column(column)
    ]
    if _blocked_outcomes:
        excluded.update(_blocked_outcomes)
        audit_log.append(
            "Blocked outcome/label column(s) from biomarker candidates (using one outcome to predict "
            f"another is construct-circular): {', '.join(map(str, _blocked_outcomes))}."
        )

    # Construct-overlap guard: when the target is a psychometric/self-report instrument, sibling
    # self-report "*_score" survey columns are criterion-contaminated (self-report predicting
    # self-report), not wearable biomarkers β€” exclude them (wearable device scores are allow-listed).
    _blocked_overlap = [
        column for column in df.columns
        if column != target_column and column not in confounders and column not in excluded
        and _is_construct_overlap_score(column, target_column)
    ]
    if _blocked_overlap:
        excluded.update(_blocked_overlap)
        audit_log.append(
            "Blocked self-report '*_score' survey column(s) as construct overlap with the self-report "
            f"target '{target_column}' (self-report predicting self-report is circular): "
            f"{', '.join(map(str, _blocked_overlap))}."
        )

    work = df.copy()
    # Aggregate to one row per participant when a time axis is present OR when a
    # participant appears in multiple rows (repeated measures). The latter guards
    # against pseudo-replication: treating non-independent rows from the same
    # person as independent observations would inflate effective n and significance.
    _has_duplicate_participants = (
        participant_id_column
        and participant_id_column in work.columns
        and int(work[participant_id_column].nunique(dropna=True)) < len(work)
    )
    # Aggregation is correct when each participant contributes repeated measures of a
    # participant-LEVEL outcome (one HOMA-IR per person, a constant diagnosis). But when
    # the TARGET itself varies WITHIN a participant (a stress label that is 0 at baseline
    # and 1 under stress; a per-visit severity score) and there is no time axis to model
    # it longitudinally, collapsing to one row per participant destroys the very signal
    # we want β€” the target would aggregate to a near-constant. In that case keep the
    # row-level data; within-participant correlation is handled by the mixed-effects model.
    _target_varies_within_participant = False
    if _has_duplicate_participants:
        _target_for_variation = pd.to_numeric(work[target_column], errors="coerce")
        _unique_per_participant = (
            work.assign(_codas_target_var=_target_for_variation)
            .groupby(participant_id_column)["_codas_target_var"]
            .nunique(dropna=True)
        )
        if len(_unique_per_participant):
            _target_varies_within_participant = float((_unique_per_participant >= 2).mean()) >= 0.20
    _should_aggregate = bool(
        participant_id_column
        and participant_id_column in work.columns
        and (time_column or _has_duplicate_participants)
    )
    if _should_aggregate and not time_column and _target_varies_within_participant:
        _should_aggregate = False
        audit_log.append(
            f"Kept row-level repeated measures: target '{target_column}' varies within "
            f"'{participant_id_column}' for a substantial fraction of participants, so participant "
            "aggregation would erase the within-person signal. Within-participant correlation is "
            "addressed by the mixed-effects model instead."
        )
    if _should_aggregate:
        work = _aggregate_longitudinal(work, target_column, participant_id_column, time_column, excluded, confounders)
        target_note = work.attrs.get("codas_target_aggregation_note", "target aggregation")
        if time_column:
            audit_log.append(
                f"Aggregated longitudinal rows by '{participant_id_column}' using mean/std/min/max/median features and {target_note}."
            )
        else:
            audit_log.append(
                f"Aggregated repeated-measures rows (no time column detected) by '{participant_id_column}' to avoid "
                f"pseudo-replication, using mean/std/min/max/median features and {target_note}."
            )
    elif participant_id_column and participant_id_column not in work.columns:
        audit_log.append(f"Participant ID column '{participant_id_column}' was not found and was ignored.")
        participant_id_column = None

    reserved = {target_column}
    if participant_id_column:
        reserved.add(participant_id_column)
    if time_column:
        reserved.add(time_column)
    reserved.update(excluded)
    reserved.update(confounders)

    feature_columns = [
        column
        for column in work.columns
        if column not in reserved
        and not _is_demographic_column(column)  # demographics/SES are confounders, not biomarker candidates
        and not _is_outcome_label_column(column)  # other outcome/label columns are circular, not candidates
        and pd.api.types.is_numeric_dtype(work[column])
    ]
    feature_columns = [
        column
        for column in feature_columns
        if work[column].notna().sum() >= 20 and work[column].nunique(dropna=True) > 1
    ]

    # Drop trivial identifier / row-index columns that survived as numeric features. A column whose
    # NAME looks like an id/index AND whose values are ~unique per row is a record id, not a
    # biomarker. Without this, a sequential id that happens to track the target (sorted exports β€”
    # the A15 trivial-leakage trap) could be screened and ratio-engineered into a spurious
    # "biomarker". Requiring BOTH the name pattern AND near-uniqueness keeps real measurements
    # (e.g. a low-cardinality "device_id" grouping, or any genuinely continuous feature).
    _ID_NAMES = {
        "id", "index", "rowid", "row_id", "row_index", "rowindex", "record_id", "recordid",
        "record_number", "uuid", "guid", "seq", "sequence", "sample_id", "sampleid", "row",
        "row_number", "rownum", "serial", "serial_number",
    }
    _AGG_SUFFIXES = ("_mean", "_median", "_min", "_max", "_std", "_sum", "_count", "_first", "_last", "_iqr", "_range")

    def _is_identifier_like(col: str) -> bool:
        # STRUCTURAL row-index/ID detector (name-independent): a column whose values are all-unique
        # CONSECUTIVE integers (an arange-like 0..n-1 / 1..n in any order) is a row index or record id,
        # never a continuous biomarker. Caught regardless of column name β€” fixes the row-order leakage
        # where a sequential id on a target-SORTED export was screened+validated as a "biomarker"
        # (found via injection probe on thyroid_recurrence / obesity_level, which are sorted by target).
        _series0 = work[col].dropna()
        _n0 = len(_series0)
        if _n0 >= 20 and _series0.nunique() == _n0:
            _vals = pd.to_numeric(_series0, errors="coerce").dropna()
            if len(_vals) == _n0 and np.all(np.isfinite(_vals)) and np.allclose(np.mod(_vals, 1.0), 0.0):
                _span = float(_vals.max() - _vals.min())
                if abs(_span - (_n0 - 1)) <= max(1.0, 0.001 * _n0):  # consecutive integer sequence
                    return True
        name = str(col).strip().lower()
        # Longitudinal aggregation renames "record_id" -> "record_id_mean"/"_min"/... so match the
        # base name (suffix stripped) as well as the raw name.
        base = name
        for suffix in _AGG_SUFFIXES:
            if base.endswith(suffix):
                base = base[: -len(suffix)]
                break
        name_hit = any(
            candidate in _ID_NAMES
            or candidate.endswith("_id") or candidate.endswith("_index")
            or candidate.endswith("_uuid") or candidate.endswith("_guid")
            or candidate.startswith("unnamed:")
            for candidate in (name, base)
        )
        if not name_hit:
            return False
        series = work[col].dropna()
        return len(series) >= 20 and series.nunique() >= 0.95 * len(series)

    identifier_like = [column for column in feature_columns if _is_identifier_like(column)]
    if identifier_like:
        feature_columns = [column for column in feature_columns if column not in identifier_like]
        audit_log.append(
            f"Excluded identifier/index-like columns from biomarker features (near-unique per row): "
            f"{', '.join(map(str, identifier_like))}."
        )
    feature_components = {column: [column] for column in feature_columns}
    feature_columns, added_ratios = _add_ratio_features(
        work,
        feature_columns,
        feature_components,
        max_ratio_features=max_ratio_features,
    )
    if added_ratios:
        audit_log.append(f"Generated {len(added_ratios)} ratio features before label-aware screening.")

    # Interaction (product) features get their OWN budget so they are never crowded out by ratios.
    try:
        added_interactions = _add_interaction_features(work, feature_columns, feature_components)
        if added_interactions:
            feature_columns = feature_columns + added_interactions
            audit_log.append(f"Generated {len(added_interactions)} interaction (product) features before screening.")
    except Exception:
        pass  # interactions are additive; never fail the frame build

    keep_columns = list(dict.fromkeys(
        [column for column in [participant_id_column, target_column] if column]
        + confounders
        + feature_columns
        + [column for column in excluded if column in work.columns]
    ))
    work = work[keep_columns].replace([np.inf, -np.inf], np.nan)
    return AnalysisFrame(
        frame=work,
        target_column=target_column,
        participant_id_column=participant_id_column,
        time_column=time_column,
        feature_columns=feature_columns,
        confounder_columns=confounders,
        excluded_columns=[column for column in excluded if column in work.columns],
        feature_components=feature_components,
        audit_log=audit_log,
    )