Spaces:
Sleeping
Sleeping
File size: 72,624 Bytes
46fe844 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 | """Dataset loading, profiling, and feature-matrix construction."""
from __future__ import annotations
import csv
from dataclasses import dataclass
from pathlib import Path
import re
import warnings
import numpy as np
import pandas as pd
from .models import DatasetProfile
class InsufficientDataError(ValueError):
"""Raised when a dataset is too sparse/degenerate for a rigorous run.
This is an expected guardrail outcome (not a bug): the agent refuses to
fabricate associations on underpowered, constant, or feature-less data.
Callers should convert it into a clear user-facing boundary message.
"""
TARGET_HINTS = (
"target",
"outcome",
"label",
"phq",
"depression",
"anxiety",
"homa",
"insulin",
"glucose",
"severity",
"score",
)
ID_HINTS = (
"participant", "subject", "subjid", "usubjid", "user_id", "person", "pid",
"patient", "record_id", "mrn", "wearer", "studyid", "study_id", "_id",
"group", # cluster/group ID (ecological studies, multi-site)
"cluster", # cluster ID
"household", "unit_id",
)
TIME_HINTS = ("time", "date", "timestamp", "datetime", "day", "hour", "week")
@dataclass
class AnalysisFrame:
frame: pd.DataFrame
target_column: str
participant_id_column: str | None
time_column: str | None
feature_columns: list[str]
confounder_columns: list[str]
excluded_columns: list[str]
feature_components: dict[str, list[str]]
audit_log: list[str]
def _detect_delimiter(path: Path) -> str:
"""Detect the field delimiter from the header line (most frequent of , ; tab |). Delimiter
chars are ASCII, so this is robust to the file's text encoding. Handles European `;` exports,
TSV, and pipe-delimited files without guessing per-row."""
try:
with open(path, encoding="utf-8", errors="replace") as fh:
line = fh.readline()
except Exception:
return ","
counts = {d: line.count(d) for d in (",", ";", "\t", "|")}
best = max(counts, key=counts.get)
return best if counts[best] > 0 else ","
def _robust_read_csv(path: Path, **kwargs) -> pd.DataFrame:
"""Read a CSV the way real users actually upload it: detect the delimiter (`;` European/TSV/
pipe, not just comma), try common encodings (Excel often exports cp1252/latin-1), handle the
European comma-decimal convention (`80,5`), and tolerate ragged rows / unbalanced quotes (skip
bad lines), so a real-world file degrades gracefully instead of crashing. Raises a clear
InsufficientDataError only when nothing parses at all."""
explicit_sep = ("sep" in kwargs) or ("delimiter" in kwargs)
sep = None if explicit_sep else _detect_delimiter(path)
def _read(enc: str, **extra) -> pd.DataFrame:
opts = dict(kwargs)
if sep is not None:
opts["sep"] = sep
opts.update(extra)
return pd.read_csv(path, encoding=enc, **opts)
def _numeric_cols(df: pd.DataFrame) -> int:
n = 0
for col in df.columns:
s = df[col]
if pd.api.types.is_numeric_dtype(s) or pd.to_numeric(s, errors="coerce").notna().mean() > 0.8:
n += 1
return n
last_err: Exception | None = None
for enc in ("utf-8", "utf-8-sig", "cp1252", "latin-1"):
try:
base = _read(enc)
except UnicodeDecodeError as exc:
last_err = exc
continue # wrong encoding -> try the next one
except pd.errors.ParserError as exc:
last_err = exc
py_kwargs = {k: v for k, v in kwargs.items() if k != "low_memory"}
for extra in ({"on_bad_lines": "skip"}, {"on_bad_lines": "skip", "quoting": csv.QUOTE_NONE}):
try:
return pd.read_csv(path, encoding=enc, sep=(sep or ","), engine="python", **py_kwargs, **extra)
except (pd.errors.ParserError, UnicodeDecodeError) as exc2:
last_err = exc2
continue
continue
# European comma-decimal: for `;`-delimited files, prefer decimal="," if it yields MORE
# numeric columns (so "80,5" parses to 80.5 instead of staying a string).
if sep == ";":
try:
euro = _read(enc, decimal=",", thousands=".")
if _numeric_cols(euro) > _numeric_cols(base):
return euro
except Exception:
pass
return base
raise InsufficientDataError(
f"Could not parse this CSV ({type(last_err).__name__ if last_err else 'unknown error'}: "
f"{str(last_err)[:140] if last_err else ''}). Check the delimiter, quoting, and encoding "
"(UTF-8 or Latin-1) β or re-export the file from your tool β and upload again."
)
# Memory guard: above this file size, read a BOUNDED row sample instead of the whole CSV, so a large
# wearable export (which on Cloud Run's in-memory tmpfs would otherwise load fully into RAM) cannot OOM
# the 8Gi instance. The cap is by CELLS (β20M), so wide files read fewer rows. The discovery engine
# further subsamples for screening and discloses it, so analysis on a representative sample stays honest.
_LARGE_FILE_BYTES = 80 * 1024 * 1024
# Keep the in-memory footprint small even for object-heavy files (e.g. wearable exports with dozens of
# repeated string/demographic columns): the engine subsamples to ~40k rows for screening anyway, so a
# ~3M-cell sample is plenty and keeps peak RSS bounded well under the 8Gi instance (which also holds the
# raw file in tmpfs during processing).
_LARGE_FILE_CELL_CAP = 3_000_000
_LARGE_FILE_MIN_ROWS = 20_000
_LARGE_FILE_CHUNK_CELLS = 600_000 # per-chunk parse footprint cap (rows = this // ncols)
def _bounded_read_rows(path: Path) -> int | None:
"""Row cap for a large file (None = read all). Reads only the header to size the cap by columns."""
try:
if path.stat().st_size <= _LARGE_FILE_BYTES:
return None
ncols = max(1, _robust_read_csv(path, nrows=1).shape[1])
return max(_LARGE_FILE_MIN_ROWS, _LARGE_FILE_CELL_CAP // ncols)
except Exception:
return None
def _count_data_rows(path: Path) -> int:
"""Fast streaming line count (O(1) memory) minus the header row."""
with open(path, "rb") as f:
n = sum(buf.count(b"\n") for buf in iter(lambda: f.read(1 << 20), b""))
return max(1, n - 1)
def _read_csv_systematic_sample(path: Path, cap_rows: int) -> pd.DataFrame:
"""Memory-bounded REPRESENTATIVE read of a large CSV. Streams the file LINE BY LINE (O(1) memory β
pandas' chunked parser still buffers GBs for a multi-GB file), keeps the header + every stride-th
data line into a small in-memory buffer, then parses only that. Spans the WHOLE file, so all
participants are represented even when rows are grouped by subject; peak memory β the sample, not
the file."""
import io
total = _count_data_rows(path)
stride = max(1, total // max(1, cap_rows))
if stride <= 1:
return _robust_read_csv(path, low_memory=False)
buf = io.StringIO()
try:
with open(path, "r", newline="", encoding="utf-8", errors="replace") as f:
header = f.readline()
if not header:
return _robust_read_csv(path, nrows=cap_rows, low_memory=False)
buf.write(header)
for i, line in enumerate(f): # i=0 is the FIRST data row
if i % stride == 0:
buf.write(line)
except Exception:
return _robust_read_csv(path, nrows=cap_rows, low_memory=False) # fall back to head sample
buf.seek(0)
try:
# on_bad_lines="skip": real wearable exports are often ragged (a few rows with an extra field).
return pd.read_csv(buf, low_memory=False, on_bad_lines="skip")
except Exception:
return _robust_read_csv(path, nrows=cap_rows, low_memory=False)
def _normalize_string_extension_dtypes(df: pd.DataFrame) -> pd.DataFrame:
"""Coerce pandas 'string'/Arrow-backed string columns (pandas 2.x ``StringDtype`` /
``ArrowStringArray``) to plain numpy ``object`` dtype.
Why: sklearn's array indexing (``train_test_split``, and ``groups=`` for
``GroupKFold``/``StratifiedGroupKFold``) cannot integer-index an Arrow-backed array and raises
``"only integer scalar arrays can be converted to a scalar index"``. Any dataset with a string
participant-id column (the common case) would otherwise crash the validation/CV pipeline. This
is generic β it inspects dtypes only, makes no dataset-specific assumption, and leaves numeric,
datetime, and already-object columns untouched."""
for col in df.columns:
dt = df[col].dtype
try:
is_str_ext = isinstance(dt, pd.StringDtype) or (
pd.api.types.is_extension_array_dtype(dt) and pd.api.types.is_string_dtype(dt)
)
except Exception:
is_str_ext = False
if is_str_ext:
df[col] = df[col].astype(object)
return df
def read_csv_dataset(path: str | Path, nrows: int | None = None, *, coerce_numeric: bool = True) -> pd.DataFrame:
"""Read a CSV with light schema repair for exported biomedical tables.
For files above ``_LARGE_FILE_BYTES`` and when the caller did not request an explicit ``nrows``, a
memory-bounded REPRESENTATIVE (systematic, whole-file) row sample is read so a multi-GB upload
cannot exhaust the instance while still covering all participants."""
path = Path(path)
if nrows is None:
_cap = _bounded_read_rows(path)
if _cap is not None:
df = _read_csv_systematic_sample(path, _cap)
df = df.loc[:, ~df.columns.astype(str).str.match(r"^Unnamed: 0$")]
return _normalize_string_extension_dtypes(_coerce_numeric_like_columns(df) if coerce_numeric else df)
df = _robust_read_csv(path, nrows=nrows, low_memory=False)
unnamed_fraction = sum(str(column).startswith("Unnamed") for column in df.columns) / max(1, len(df.columns))
if unnamed_fraction > 0.35:
probe = _robust_read_csv(path, header=None, nrows=3)
second_row = probe.iloc[1].dropna().astype(str).str.lower().tolist() if len(probe) > 1 else []
header_like = any(token in " ".join(second_row) for token in ("participant", "homa", "label", "heart", "steps"))
if header_like:
df = _robust_read_csv(path, header=1, nrows=nrows, low_memory=False)
df = df.loc[:, ~df.columns.astype(str).str.match(r"^Unnamed: 0$")]
if coerce_numeric:
df = _coerce_numeric_like_columns(df)
return _normalize_string_extension_dtypes(df)
_ID_KEY_PRIORITY = (
"participant_id",
"subject_id",
"user_id",
"person_id",
"patient_id",
"pid",
"id",
)
def _short_source_name(name: str) -> str:
base = Path(str(name or "dataset")).stem
safe = "".join(ch if ch.isalnum() else "_" for ch in base).strip("_").lower()
return (safe or "dataset")[:24]
def _detect_join_key(frames: list[pd.DataFrame]) -> str | None:
"""Pick an ID-like column present in *every* frame, by priority then by name pattern."""
common = set(frames[0].columns)
for frame in frames[1:]:
common &= set(frame.columns)
if not common:
return None
lowered = {str(col).lower(): col for col in common}
for key in _ID_KEY_PRIORITY:
if key in lowered:
return lowered[key]
for low, original in lowered.items():
if low.endswith("_id") or low == "id" or "participant" in low or "subject" in low:
return original
return None
def merge_datasets(named_frames: list[tuple[str, pd.DataFrame]]) -> tuple[pd.DataFrame, dict]:
"""Combine several uploaded CSVs into one analysis frame so the agent can reason across
them jointly.
Strategy:
* **join** on a shared ID column (e.g. participant_id) when one exists in every file β
the common case of "wearable features in one file, clinical labels in another".
* **union** (stack rows) when the files share an identical schema but no ID key.
* otherwise raise InsufficientDataError so the caller can explain the boundary rather
than silently producing a meaningless cross-product.
Colliding non-key column names are prefixed with the source's short name so provenance is
preserved and nothing is silently overwritten.
"""
named_frames = [(name, frame) for name, frame in named_frames if frame is not None and len(frame.columns)]
if len(named_frames) < 2:
raise InsufficientDataError("merge_datasets needs at least two datasets.")
names = [_short_source_name(name) for name, _ in named_frames]
# Disambiguate duplicate source names so prefixes stay unique.
seen: dict[str, int] = {}
for index, name in enumerate(names):
if name in seen:
seen[name] += 1
names[index] = f"{name}_{seen[name]}"
else:
seen[name] = 0
frames = [frame for _, frame in named_frames]
sources = [
{"name": names[i], "rows": int(len(frames[i])), "columns": int(frames[i].shape[1])}
for i in range(len(frames))
]
key = _detect_join_key(frames)
if key is not None:
# Prefix only the columns that collide across files (keep clean names otherwise).
non_key_counts: dict[str, int] = {}
for frame in frames:
for col in frame.columns:
if col != key:
non_key_counts[col] = non_key_counts.get(col, 0) + 1
colliding = {col for col, count in non_key_counts.items() if count > 1}
renamed: dict[str, str] = {}
prepared = []
for source_name, frame in zip(names, frames):
frame = frame.copy()
frame[key] = frame[key].astype(str)
rename_map = {}
for col in frame.columns:
if col != key and col in colliding:
new_col = f"{source_name}_{col}"
rename_map[col] = new_col
renamed[f"{source_name}.{col}"] = new_col
if rename_map:
frame = frame.rename(columns=rename_map)
prepared.append(frame)
# Cardinality / row-explosion detection: a one-to-many or many-to-many join REPLICATES
# rows from the smaller file(s), so the merged rows are NOT independent observations.
# Surfacing this prevents silent pseudo-replication (inflated n / significance).
per_source_card = []
sources_with_dupe_keys = 0
for src_name, frame in zip(names, prepared):
vc = frame[key].value_counts()
max_per_key = int(vc.max()) if len(vc) else 0
per_source_card.append({
"name": src_name, "rows": int(len(frame)), "unique_keys": int(frame[key].nunique()),
"max_rows_per_key": max_per_key, "duplicate_key_rate": round(float((vc > 1).mean()) if len(vc) else 0.0, 3),
})
if max_per_key > 1:
sources_with_dupe_keys += 1
merged = prepared[0]
for frame in prepared[1:]:
merged = merged.merge(frame, on=key, how="outer")
rows_before = sum(int(len(f)) for f in prepared)
largest = max((int(len(f)) for f in prepared), default=0)
rows_after = int(len(merged))
cardinality = "many_to_many" if sources_with_dupe_keys >= 2 else ("one_to_many" if sources_with_dupe_keys == 1 else "one_to_one")
key_sets = [set(f[key].dropna().unique()) for f in prepared]
common = set.intersection(*key_sets) if key_sets else set()
all_keys = set().union(*key_sets) if key_sets else set()
unmatched_keys = int(len(all_keys) - len(common))
explosion_warning = None
if cardinality != "one_to_one":
explosion_warning = (
f"{cardinality.replace('_', '-')} join on '{key}': rows from the smaller file(s) are REPLICATED "
f"across matches (merged to {rows_after:,} rows from sources of "
f"{', '.join(str(c['rows']) for c in per_source_card)}). These {rows_after:,} rows are NOT "
f"independent observations β treat '{key}' as a participant/cluster id (mixed-effects or "
f"participant-level split), otherwise per-row significance is inflated."
)
info = {
"strategy": "join",
"join_key": key,
"sources": sources,
"rows": rows_after,
"rows_before_join": rows_before,
"columns": int(merged.shape[1]),
"renamed_columns": renamed,
"cardinality": cardinality,
"row_multiplier": round(rows_after / largest, 2) if largest else None,
"per_source_key_cardinality": per_source_card,
"unmatched_keys": unmatched_keys,
"explosion_warning": explosion_warning,
}
return merged, info
# No shared key: only safe to union if every file has the same column set.
column_sets = [tuple(frame.columns) for frame in frames]
if len(set(column_sets)) == 1:
stacked = pd.concat(
[frame.assign(__source=name) for name, frame in zip(names, frames)],
ignore_index=True,
)
info = {
"strategy": "union",
"join_key": None,
"sources": sources,
"rows": int(len(stacked)),
"columns": int(stacked.shape[1]),
"renamed_columns": {},
}
return stacked, info
raise InsufficientDataError(
"These datasets don't share an ID column (e.g. participant_id) to join on, and their "
"columns differ, so they can't be combined automatically. Give them a common ID column, "
"or analyze one dataset at a time."
)
def _coerce_numeric_like_columns(df: pd.DataFrame) -> pd.DataFrame:
"""Promote numeric-looking object columns without touching datetime-like text.
User uploads often contain commas, percent signs, or currency-like prefixes that
make pandas treat otherwise numeric sensor/lab columns as strings. CoDaS should
still audit and model those columns, while preserving dates and free text.
"""
work = df.copy()
for column in work.columns:
series = work[column]
if not pd.api.types.is_object_dtype(series) and not pd.api.types.is_string_dtype(series):
continue
if _looks_like_datetime(series):
continue
cleaned = (
series.astype("string")
.str.replace(",", "", regex=False)
.str.replace("%", "", regex=False)
.str.replace("$", "", regex=False)
.str.strip()
)
observed = cleaned.notna() & (cleaned != "")
if int(observed.sum()) < 20:
continue
numeric = pd.to_numeric(cleaned, errors="coerce")
if float(numeric.notna().sum() / max(1, observed.sum())) >= 0.8:
work[column] = numeric
return work
def _looks_like_datetime(series: pd.Series) -> bool:
if pd.api.types.is_datetime64_any_dtype(series):
return True
if not pd.api.types.is_object_dtype(series):
return False
sample = series.dropna().astype(str).head(30)
if sample.empty:
return False
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
parsed = pd.to_datetime(sample, errors="coerce")
return parsed.notna().mean() >= 0.8
def _name_tokens(name: str) -> set[str]:
"""Lowercased alphanumeric tokens of a column name, split on any non-alnum boundary.
'f_slp:..._rapids_...' -> {'f','slp',...,'rapids',...}; 'participant_id' -> {'participant','id'}."""
norm = "".join(ch.lower() if ch.isalnum() else " " for ch in str(name))
return {t for t in norm.split() if t}
def _hinted(columns: list[str], hints: tuple[str, ...]) -> list[str]:
"""Columns whose NAME TOKENS contain a hint. Token-based (not substring) so an identifier
hint like 'pid' matches a 'pid' / 'participant_pid' column but NOT 'f_..._rapids_...' β the
GLOBEM/RAPIDS feature family, where 'rapids' embeds the substring 'pid'. A multi-token hint
('record_id', 'user_id') matches only when ALL its tokens are present. (Substring matching
here false-matched 3,591 RAPIDS features and collapsed 8,225 rows into 2 'participants'.)"""
hint_tokens = [tuple(_name_tokens(h)) for h in hints]
out: list[str] = []
for column in columns:
toks = _name_tokens(column)
if any(ht and set(ht).issubset(toks) for ht in hint_tokens):
out.append(column)
return out
def _detect_participant_column(df: pd.DataFrame, target_column: str | None = None) -> str | None:
"""Detect the participant / cluster identifier from column names + structure.
Repeated-measures rows from the same person are NOT independent: treating them as
independent inflates effective n and significance (pseudo-replication). The participant
column drives aggregation/mixed-effects, so missing it is a high-impact correctness bug.
Name detection alone is fragile -- clinical & standard identifiers (record_id, patient_id,
usubjid, mrn, bare 'id') are not in the original hint list -- so we (1) match an expanded
clinical/standard hint list, preferring a column that actually clusters (repeats), then
(2) fall back to any identifier-NAMED column (id / *_id / *code / *key) that clusters.
Detection stays high-precision (identifier-named only) to avoid mis-aggregating a real
low-cardinality feature (hour, site, num_visits) as if it were the subject id.
"""
n = int(len(df))
if n == 0:
return None
cols = [c for c in df.columns if c != target_column]
def k_of(column: str) -> int:
try:
return int(df[column].nunique(dropna=True))
except Exception:
return n
# (1) Name-hinted identifiers (expanded). Prefer one that genuinely clusters rows; among
# several, the participant/cluster unit is the one with the FEWEST groups (most rows each),
# so a near-unique sample_id never wins over a repeating subject id. BUT a canonical SUBJECT
# identifier (participant_id / subject_id / *_id) must outrank a generic cluster/category hint
# (group / cluster / household / unit_id) β otherwise a low-cardinality demographic CATEGORY
# (e.g. a 3-value 'household' = lives-alone/with-others) wrongly beats a real 'participant_id'
# by having the fewest groups, mis-aggregating millions of rows into a handful of "participants".
def _id_priority(column: str) -> int:
toks = _name_tokens(column) # token-based (not substring): 'pid' must not score 'rapids'
if toks & {"participant", "subject", "subjid", "usubjid", "wearer"}:
return 0 # canonical per-person identifier
if (toks & {"person", "patient", "pid", "mrn", "studyid"}
or {"user", "id"} <= toks or {"record", "id"} <= toks or {"study", "id"} <= toks):
return 1 # other per-person identifier
if "id" in toks:
return 2 # generic *_id (could be a record/sample id)
return 3 # generic cluster/category hint: group, cluster, household, unit_id
hinted = _hinted(cols, ID_HINTS)
if hinted:
clustering = [c for c in hinted if 1 < k_of(c) < n]
pool = clustering or hinted
best_pri = min(_id_priority(c) for c in pool)
tier = [c for c in pool if _id_priority(c) == best_pri]
# within the strongest name tier, the FEWEST-groups column is the subject/cluster unit
return min(tier, key=k_of)
# (2) Structural fallback: an identifier-NAMED column that clusters (>= 2 rows per id).
structural = []
for c in cols:
name = str(c).strip().lower()
idish = (
name in ("id", "uid", "uuid", "guid")
or name.endswith("_id")
or name.endswith("code")
or name.endswith("key")
)
if not idish:
continue
k = k_of(c)
if 1 < k < n and (n / k) >= 2:
structural.append(c)
if structural:
return min(structural, key=k_of)
return None
# Demographic / SES tokens β these are CONFOUNDERS or keys, never wearable biomarker CANDIDATES
# (e.g. has_disability, education, income, marital_status). They may still be adjusted for as
# confounders, but must not be promoted as "discovered biomarkers". Token-based (split on non-alnum)
# so "age" can't hit "usage"/"average" and "race" can't hit "embrace".
_DEMOGRAPHIC_FEATURE_TOKENS = frozenset({
"age", "gender", "sex", "race", "ethnicity", "ethnic", "hispanic", "latinx", "marital",
"education", "educational", "income", "financial", "occupation", "employment", "disability",
"disabled", "household", "demographic", "demographics", "socioeconomic", "ses",
# marital-status family: a real dwb run surfaced `is_married` (Ο=-0.33) as an "exploratory"
# candidate because only "marital" was a token. Marital status is a SOCIAL confounder of
# depression, not a wearable biomarker. High-precision tokens only (NOT "single", which would
# false-match wearable features like single_bout / single_use).
"married", "unmarried", "marriage", "spouse", "spousal", "divorced", "widowed", "widow",
})
def _is_demographic_column(column: str) -> bool:
"""True for demographic/SES columns that must not be promoted as biomarker candidates."""
norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
toks = {t for t in norm.split("_") if t}
return bool(toks & _DEMOGRAPHIC_FEATURE_TOKENS)
# Outcome / label / clinical-endpoint tokens β these are OUTCOMES, never biomarker CANDIDATES.
# Using one outcome measure to "predict" another operationalization of the same construct is
# construct circularity / label leakage. On GLOBEM the chosen target is `target_depression_weekly`
# while `target_BDI2`, `target_phq4`, `target_dep`, `target_depression_merged`, `target_anxiety_weekly`
# are ALTERNATE depression/anxiety labels β predicting depression from another depression scale is a
# credibility disaster ("depression predicts depression"). Such columns can be the TARGET or a
# covariate, but must never surface as a discovered "biomarker". HIGH-PRECISION token set: explicit
# label markers + validated psychiatric/clinical instrument acronyms ONLY β deliberately NOT broad
# tokens like "score"/"severity"/"glucose"/"insulin" that are legitimate sensor biomarkers elsewhere.
_OUTCOME_LABEL_TOKENS = frozenset({
"target", "targets", "outcome", "outcomes", "label", "labels", "groundtruth", "endpoint",
"diagnosis", "diagnosed", "diagnostic",
"phq", "phq2", "phq4", "phq8", "phq9", "gad", "gad2", "gad7", "bdi", "bdi2", "cesd", "cesd10",
"pss", "panas", "stai", "ders", "ucla", "psqi", "who5",
})
def _is_outcome_label_column(column: str) -> bool:
"""True for outcome/label/clinical-endpoint columns that must not be biomarker candidates
(they may be the target or a covariate, but predicting one outcome from another is circular).
Token-based + aggregation-suffix robust ('target_BDI2_mean' -> {target,bdi2,mean} matches)."""
norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
toks = {t for t in norm.split("_") if t}
return bool(toks & _OUTCOME_LABEL_TOKENS)
# Psychometric / self-report target tokens. When the TARGET is one of these (or any "*_score"
# self-report instrument), sibling "*_score" SURVEY columns are CONSTRUCT OVERLAP β self-report
# predicting self-report, not a wearable biomarker. On dwb, predicting phq_score (PHQ-9, which itself
# contains sleep & fatigue items) from sleep_disturbance_score / tiredness_score / sleep_impairment_score
# is criterion contamination ("depression symptoms predict depression"). The agent layer
# (orchestration.infer_exclusions) already strips these for the UI; mirroring it here makes the CORE
# ENGINE self-protecting so a direct run_discovery / API / harness call gets the same guard.
_PSYCHOMETRIC_TARGET_TOKENS = frozenset({
"phq", "gad", "pss", "depress", "depression", "anxiety", "anxious", "mood", "stress",
"wellbe", "wellbeing", "affect", "psych", "panas", "who5", "cesd", "bdi", "stai", "ders",
})
# Wearable / device-derived scores that ARE legitimate candidates even against a psychometric target.
_WEARABLE_SCORE_ALLOWLIST = (
"step", "heart_rate", "resting_heart", "hrv", "unlock", "screen", "accel", "bins",
"min_in_", "distance", "calorie", "wear", "sleep_efficiency", "activity", "cadence",
)
def _target_is_psychometric(target_column: str | None) -> bool:
if not target_column:
return False
norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(target_column)).strip("_")
toks = {t for t in norm.split("_") if t}
return norm.endswith("_score") or norm == "score" or bool(toks & _PSYCHOMETRIC_TARGET_TOKENS)
def _is_construct_overlap_score(column: str, target_column: str | None) -> bool:
"""A self-report '*_score' SURVEY sibling of a psychometric/self-report target is construct
overlap (self-report predicting self-report), not a wearable biomarker. Wearable device scores
(sleep_efficiency, step, heart_rate, ...) are allow-listed. Operates on RAW column names (run
in build_analysis_frame BEFORE longitudinal aggregation, so 'sleep_disturbance_score' is caught
before it becomes 'sleep_disturbance_score_mean')."""
if not target_column or column == target_column:
return False
norm = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
if not (norm.endswith("_score") or norm == "score"):
return False
if any(tok in norm for tok in _WEARABLE_SCORE_ALLOWLIST):
return False
return _target_is_psychometric(target_column)
def _suggested_target_columns(columns: list[str], numeric_columns: list[str]) -> list[str]:
"""Rank plausible endpoints ahead of generic demographic survey scores."""
exact_priority = {
"depression_status": 130,
"depression_binary": 125,
"madrs_start": 120,
"madrs_end": 115,
"madrs_change": 110,
"true_homa_ir": 100,
"homa_ir": 100,
"depression_score": 100,
"phq_score": 100,
"phq_total": 100,
"insulin_resistance_score": 100,
"relapse_label": 100,
"fatigue_score": 100,
"outcome_severity_score": 100,
}
domain_tokens = (
"homa",
"insulin",
"glucose",
"hba1c",
"phq",
"madrs",
"depression",
"anxiety",
"gad",
"relapse",
"fatigue",
"outcome",
"target",
"label",
"severity",
# psychometric / clinical endpoints common in wearable + EMA studies
"pss",
"stress",
"mood",
"wellbeing",
"mortality",
"death",
"diagnosis",
"disease",
"dementia",
"updrs",
)
# Demographic / SES / identifier tokens are NEVER a discovery endpoint (they are confounders or
# keys). Token-based exclusion so "age" doesn't match "usage" and "race" doesn't match "embrace".
nonoutcome_tokens = {
"age", "gender", "sex", "race", "ethnicity", "ethnic", "hispanic", "latinx", "marital",
"education", "income", "financial", "occupation", "employment", "disability", "household",
"id", "participant", "subject", "user", "pid", "mrn", "uuid",
# Provenance, study-design, and sensor-placement descriptors can mention a disease or
# endpoint domain without being an analyzable outcome.
"context", "source", "dataset", "study", "cohort", "site", "center", "protocol",
"placement", "placements", "sensor", "sensors", "annotation", "annotations",
"provenance", "metadata", "device", "firmware",
}
generic_penalties = (
"age_score",
"gender_score",
"sex_score",
"race",
"ethnicity",
"financial",
"income",
"bfi",
"intake_survey",
"duration_score",
"composition_score",
)
scored: list[tuple[int, int, str]] = []
for column in columns:
normalized = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
toks = {t for t in normalized.split("_") if t}
# Exclude demographics/SES/ids and any time axis from "likely targets" β these horrified the
# user when surfaced (e.g. "age" as a likely target on a 4,710-participant wearable file).
if toks & nonoutcome_tokens or _likely_time_column_name(column):
continue
score = 0
if normalized in exact_priority:
score += exact_priority[normalized]
score += sum(12 for token in domain_tokens if token in normalized)
if any(token in normalized for token in generic_penalties):
score -= 30
# NOTE: no blanket "+3 because numeric". A suggested target must carry a real endpoint signal
# (exact-priority name or a clinical/psychometric token); otherwise sensor features (steps,
# sleep minutes) and survey items masquerade as outcomes. profile_dataframe falls back to
# numeric_columns[:5] only when NO endpoint-signal column exists, so token-less labels still surface.
if score > 0:
scored.append((score, -len(str(column)), column))
scored.sort(reverse=True)
return [column for _, _, column in scored]
def _likely_time_column_name(column: str) -> bool:
"""Return true for actual time axes, not summary features like weekly means."""
name = str(column).lower()
normalized = "".join(ch.lower() if ch.isalnum() else "_" for ch in str(column)).strip("_")
summary_tokens = ("mean", "median", "std", "min", "max", "weekly", "hourly", "7day")
if any(token in normalized.split("_") for token in summary_tokens):
return False
exact = {"date", "day", "hour", "week", "time", "timestamp", "datetime", "study_week"}
if normalized in exact:
return True
suffixes = ("_date", "_day", "_hour", "_week", "_time", "_timestamp", "_datetime")
prefixes = ("date_", "day_", "hour_", "week_", "time_", "timestamp_", "datetime_")
if normalized.endswith(suffixes) or normalized.startswith(prefixes):
return True
return bool(
any(token in name for token in ("datetime", "timestamp"))
or ("date" in name and not any(token in name for token in ("duration", "update")))
)
def profile_dataframe(
df: pd.DataFrame,
target_column: str | None = None,
participant_id_column: str | None = None,
time_column: str | None = None,
) -> DatasetProfile:
numeric_columns = [column for column in df.columns if pd.api.types.is_numeric_dtype(df[column])]
datetime_columns = [column for column in df.columns if _looks_like_datetime(df[column])]
categorical_columns = [
column
for column in df.columns
if column not in numeric_columns and column not in datetime_columns
]
suggested_targets = _suggested_target_columns(list(df.columns), numeric_columns)
if participant_id_column is None:
participant_id_column = _detect_participant_column(df, target_column)
if time_column is None:
time_candidates = [column for column in df.columns if _likely_time_column_name(str(column))]
time_column = time_candidates[0] if time_candidates else None
missing_fraction = {
str(column): float(df[column].isna().mean())
for column in df.columns
}
return DatasetProfile(
rows=int(len(df)),
columns=int(len(df.columns)),
numeric_columns=numeric_columns,
categorical_columns=categorical_columns,
datetime_columns=datetime_columns,
missing_fraction=missing_fraction,
suggested_targets=suggested_targets,
target_column=target_column,
participant_id_column=participant_id_column,
time_column=time_column,
)
def _add_temporal_features(
work: pd.DataFrame,
numeric: list[str],
participant_id_column: str,
time_column: str,
out_index: pd.Index,
) -> pd.DataFrame | None:
"""Per-participant temporal features that summary statistics cannot capture:
* `<feat>_trendslope` β OLS slope of the feature vs time (within-person trend; S16).
* `<feat>_weekend_minus_weekday` β weekend-mean minus weekday-mean (social-jetlag style; S15).
* `<feat>_night_over_day` β night (00-06h) over day mean ratio (circadian; S14), only when
the timestamp has sub-daily resolution.
All are computed vectorised per feature (no per-group Python loop) and bounded by a budget so a
very wide/long file degrades gracefully to summary stats only.
"""
parsed = pd.to_datetime(work[time_column], errors="coerce")
if parsed.notna().mean() < 0.6:
return None
n_participants = work[participant_id_column].nunique(dropna=True)
# Budget: skip the enhancement on very large feature*participant grids (it is additive, not core).
if not numeric or n_participants < 2 or (len(numeric) * max(1, n_participants)) > 6_000_000:
return None
tmp = work[[participant_id_column] + numeric].copy()
# Continuous time in days-from-first (works for daily and hourly cadence alike).
t0 = parsed.min()
tmp["_t"] = (parsed - t0) / pd.Timedelta(days=1)
tmp["_hour"] = parsed.dt.hour
tmp["_dow"] = parsed.dt.dayofweek # 0=Mon .. 6=Sun
gid = tmp[participant_id_column]
new_cols: dict[str, pd.Series] = {}
# (1) Trend slope: cov(t,x)/var(t) within participant β vectorised via centered sums.
t_centered = tmp["_t"] - gid.map(tmp.groupby(participant_id_column)["_t"].mean())
t_var = (t_centered ** 2).groupby(gid).sum()
valid_slope = t_var > 0
for col in numeric:
x_centered = tmp[col] - gid.map(tmp.groupby(participant_id_column)[col].mean())
num = (t_centered * x_centered).groupby(gid).sum()
slope = (num / t_var.where(valid_slope)).reindex(out_index)
new_cols[f"{col}_trendslope"] = slope
# (2) Weekday vs weekend contrast (needs both present across the dataset).
has_weekend = bool((tmp["_dow"] >= 5).any() and (tmp["_dow"] < 5).any())
if has_weekend:
wk_mask = tmp["_dow"] < 5
for col in numeric:
wk = tmp.loc[wk_mask].groupby(participant_id_column)[col].mean()
we = tmp.loc[~wk_mask].groupby(participant_id_column)[col].mean()
new_cols[f"{col}_weekend_minus_weekday"] = (we - wk).reindex(out_index)
# (3) Circadian night/day ratio (needs sub-daily resolution: >3 distinct hours).
has_subdaily = int(tmp["_hour"].nunique(dropna=True)) > 3
if has_subdaily:
night_mask = tmp["_hour"] < 6
for col in numeric:
night = tmp.loc[night_mask].groupby(participant_id_column)[col].mean()
day = tmp.loc[~night_mask].groupby(participant_id_column)[col].mean()
new_cols[f"{col}_night_over_day"] = (night / day.replace(0, np.nan)).reindex(out_index)
if not new_cols:
return None
out = pd.DataFrame(new_cols, index=out_index)
# Drop columns that are entirely NaN or constant (no information / would just be noise).
keep = [c for c in out.columns if out[c].notna().sum() >= 3 and out[c].nunique(dropna=True) > 1]
return out[keep] if keep else None
def _aggregate_longitudinal(
df: pd.DataFrame,
target_column: str,
participant_id_column: str,
time_column: str | None,
excluded_columns: set[str],
confounder_columns: list[str] | None = None,
) -> pd.DataFrame:
reserved = {target_column, participant_id_column}
if time_column:
reserved.add(time_column)
confounders = [column for column in (confounder_columns or []) if column in df.columns]
reserved.update(confounders)
work = df.copy()
for column in list(work.columns):
if column in reserved or column in excluded_columns:
continue
clock_feature = _clock_hour_feature(work[column], str(column))
if clock_feature is not None:
work[f"{column}_clock_hour"] = clock_feature
numeric = [
column
for column in work.columns
if column not in reserved
and column not in excluded_columns
and not _is_demographic_column(column) # demographics are confounders, not biomarker candidates
and not _is_outcome_label_column(column) # other outcome/label columns are circular, not candidates
and pd.api.types.is_numeric_dtype(work[column])
]
if not numeric:
raise ValueError("No numeric sensor columns are available for longitudinal aggregation.")
aggregations = {column: ["mean", "std", "min", "max", "median"] for column in numeric}
feature_df = work.groupby(participant_id_column).agg(aggregations)
feature_df.columns = [f"{name}_{agg}" for name, agg in feature_df.columns]
# Build all coefficient-of-variation columns in ONE concat. Inserting them one-by-one
# (feature_df[cv]=...) fragments the already-wide aggregate frame β O(cols^2) and a flood of
# pandas PerformanceWarnings on a 100+-column wearable file, which measurably slows feature
# engineering. Values are identical; only the assembly is vectorised.
cv_data = {
f"{column}_cv": feature_df[f"{column}_std"] / feature_df[f"{column}_mean"].replace(0, np.nan).abs()
for column in numeric
}
cv_columns = list(cv_data.keys())
if cv_data:
feature_df = pd.concat([feature_df, pd.DataFrame(cv_data, index=feature_df.index)], axis=1)
# --- Temporal feature engineering (S14/S15/S16) ---
# Summary stats (mean/std/min/max/median/cv) alone miss biomarkers whose signal lives in the
# TIME STRUCTURE: a within-person trend/slope, a weekday-vs-weekend contrast, or a circadian
# (time-of-day) pattern. When a parseable time column exists, engineer per-participant temporal
# features so these signals enter screening instead of being invisible. Bounded by a budget and
# wrapped in try/except so it can never break a discovery (degrades to summary stats only).
if time_column and time_column in work.columns:
try:
# Guard: skip temporal feature engineering for columns that look like outcome
# proxies or label derivatives. Without this guard, a near-perfect outcome proxy
# (e.g. daily_mood_selfreport, Οβ0.999 with the target) would be blocked by the
# construct-validity gate in raw form β but its temporal derivative (trendslope,
# Οβ0.26) would slip through the same gate and get validated, which is circular.
# Strategy: also exclude columns that are strongly correlated with the target
# in the aggregate frame (|rho| >= 0.95 at the participant level).
_temporal_numeric = [col for col in numeric if col not in excluded_columns]
if _temporal_numeric and target_column in df.columns:
try:
# Block temporal feature engineering for construct-circular raw features.
# Compute participant-level aggregate correlation with the target for each
# candidate raw feature and skip those with |rho| >= 0.95 (near-identical
# to the target β any temporal derivative is equally circular).
_target_agg = df.groupby(participant_id_column)[target_column].median()
from .statistics import safe_spearman as _ss
_construct_blocked: set[str] = set()
for _col in list(_temporal_numeric):
if _col not in df.columns:
continue
_col_agg = pd.to_numeric(df[_col], errors="coerce")
if _col_agg.notna().sum() < 10:
continue
_col_part = df.groupby(participant_id_column)[_col].mean()
_r, _, _ = _ss(_col_part.reindex(_target_agg.index), _target_agg)
import numpy as _np2
if _np2.isfinite(_r) and abs(_r) >= 0.95:
_construct_blocked.add(_col)
if _construct_blocked:
_temporal_numeric = [col for col in _temporal_numeric if col not in _construct_blocked]
except Exception:
pass # guard is best-effort; never fail the core aggregation
_temporal = _add_temporal_features(
work, _temporal_numeric, participant_id_column, time_column, feature_df.index,
)
if _temporal is not None and not _temporal.empty:
feature_df = pd.concat([feature_df, _temporal], axis=1)
except Exception:
pass # temporal features are an enhancement; never fail the core aggregation
target_values = df[target_column].dropna()
if target_values.nunique(dropna=True) == 2:
target_df = df.groupby(participant_id_column)[target_column].max().rename(target_column)
audit_target_note = "max/event aggregation for binary target"
else:
target_df = df.groupby(participant_id_column)[target_column].median().rename(target_column)
audit_target_note = "median aggregation for continuous target"
confounder_df = None
if confounders:
confounder_parts = []
grouped = work.groupby(participant_id_column)
for column in confounders:
if pd.api.types.is_numeric_dtype(df[column]):
confounder_parts.append(grouped[column].median().rename(column))
else:
confounder_parts.append(grouped[column].agg(lambda values: values.dropna().iloc[0] if values.dropna().size else np.nan).rename(column))
confounder_df = pd.concat(confounder_parts, axis=1) if confounder_parts else None
merged = feature_df.join(target_df)
if confounder_df is not None:
merged = merged.join(confounder_df)
merged = merged.reset_index()
merged.attrs["codas_target_aggregation_note"] = audit_target_note
return merged
def _clock_hour_feature(series: pd.Series, column_name: str) -> pd.Series | None:
"""Extract a time-of-day hour feature from sleep/wake timestamp columns."""
name = column_name.lower()
if not any(token in name for token in ("sleep", "bed", "wake", "start", "end", "left", "entered")):
return None
if not any(token in name for token in ("time", "datetime", "timestamp")):
return None
if pd.api.types.is_numeric_dtype(series):
return None
sample = series.dropna().astype(str).head(40)
if sample.empty:
return None
# Suppress pandas' per-call "Could not infer format" UserWarning. On ULTRA-WIDE files
# (GLOBEM: 5,529 cols, many object "..._sleep_..._starttime_..." columns pass the name gate)
# this function is called per column, and emitting+formatting thousands of identical warnings to
# stderr is itself an I/O bottleneck that stalled discovery (>9 min). Parsing stays correct.
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
parsed = pd.to_datetime(sample, errors="coerce")
if parsed.notna().mean() < 0.65:
return None
parsed_all = pd.to_datetime(series.astype("string"), errors="coerce")
return parsed_all.dt.hour + parsed_all.dt.minute / 60.0 + parsed_all.dt.second / 3600.0
def _feature_match_score(column: str, include: tuple[str, ...], exclude: tuple[str, ...] = ()) -> int:
# Short clinical tokens (<=3 chars: ast, alt, hdl, hrv, rhr) must match a WHOLE word-part, not an
# arbitrary substring β otherwise 'ast'/'alt' match the fetal-cardiotocography columns 'astv'/'altv'
# (abnormal short-/long-term variability) and CoDaS fabricates a nonsensical liver De-Ritis AST/ALT
# ratio from fetal heart-rate features. Longer tokens keep substring matching (e.g. 'chol' in
# 'cholesterol', 'albumin' in 'serum_albumin'), so legitimate detections (incl. ILPD sgot_ast/sgpt_alt)
# are unchanged.
normalized = column.lower()
spaced = " " + re.sub(r"[^a-z0-9]+", " ", normalized).strip() + " "
def _hit(token: str) -> bool:
t = token.lower()
if len(t) <= 3:
return (" " + t + " ") in spaced
return t in normalized
if any(_hit(token) for token in exclude):
return 0
return sum(1 for token in include if _hit(token))
def _best_feature(columns: list[str], include: tuple[str, ...], exclude: tuple[str, ...] = ()) -> str | None:
scored = [
(_feature_match_score(column, include, exclude), -len(str(column)), column)
for column in columns
]
scored = [item for item in scored if item[0] > 0]
if not scored:
return None
scored.sort(reverse=True)
return scored[0][2]
def _add_named_ratio(
frame: pd.DataFrame,
numerator: str | None,
denominator: str | None,
ratio_name: str,
feature_components: dict[str, list[str]],
added: list[str],
) -> None:
if not numerator or not denominator or numerator == denominator or ratio_name in frame.columns:
return
if numerator not in frame.columns or denominator not in frame.columns:
return
den = pd.to_numeric(frame[denominator], errors="coerce").replace(0, np.nan)
num = pd.to_numeric(frame[numerator], errors="coerce")
values = num / den
if values.notna().sum() < 20 or values.nunique(dropna=True) <= 1:
return
frame[ratio_name] = values
feature_components[ratio_name] = [numerator, denominator]
added.append(ratio_name)
def _add_domain_ratio_features(
frame: pd.DataFrame,
feature_columns: list[str],
feature_components: dict[str, list[str]],
max_ratio_features: int,
) -> list[str]:
"""Add paper-aligned candidate families before generic ratio enumeration."""
added: list[str] = []
columns = list(feature_columns)
lower_to_original = {str(column).lower(): column for column in columns}
exact = lambda name: lower_to_original.get(name.lower())
steps = _best_feature(columns, ("steps",), ("std", "cv", "min", "max"))
rhr = _best_feature(columns, ("resting heart rate",), ("std", "cv", "min", "max")) or _best_feature(columns, ("rhr",), ("std", "cv", "min", "max"))
hrv = _best_feature(columns, ("hrv",), ("std", "cv", "min", "max")) or _best_feature(columns, ("rmssd",), ("std", "cv", "min", "max"))
ast = exact("ast") or _best_feature(columns, ("ast",), ())
alt = exact("alt") or _best_feature(columns, ("alt",), ())
albumin = _best_feature(columns, ("albumin",), ())
globulin = _best_feature(columns, ("globulin",), ())
cholesterol = _best_feature(columns, ("cholesterol",), ("hdl", "ldl")) or _best_feature(columns, ("chol",), ("hdl", "ldl"))
hdl = exact("hdl") or _best_feature(columns, ("hdl",), ())
proposals = [
(steps, rhr, "cardiovascular_fitness_index_steps_over_resting_heart_rate"),
(ast, alt, "derived_ast_over_alt_ratio_de_ritis"),
(albumin, globulin, "derived_albumin_over_globulin_ratio"),
(cholesterol, hdl, "derived_cholesterol_over_hdl_ratio"),
(hrv, rhr, "derived_hrv_over_resting_heart_rate_ratio"),
]
for numerator, denominator, name in proposals:
if len(added) >= max_ratio_features:
break
_add_named_ratio(frame, numerator, denominator, name, feature_components, added)
return added
def _add_ratio_features(
frame: pd.DataFrame,
feature_columns: list[str],
feature_components: dict[str, list[str]],
max_ratio_features: int,
) -> tuple[list[str], list[str]]:
added: list[str] = []
if max_ratio_features <= 0:
return feature_columns, added
domain_added = _add_domain_ratio_features(
frame,
feature_columns,
feature_components,
max_ratio_features=max_ratio_features,
)
added.extend(domain_added)
remaining_slots = max(0, max_ratio_features - len(added))
if remaining_slots <= 0:
return feature_columns + added, added
usable = []
for column in feature_columns:
values = pd.to_numeric(frame[column], errors="coerce")
finite = values[np.isfinite(values)]
if len(finite) < 20:
continue
if (finite.abs() > 1e-9).mean() < 0.95:
continue
usable.append(column)
usable = usable[:12]
for i, numerator in enumerate(usable):
for denominator in usable[i + 1 :]:
if len(added) >= max_ratio_features:
return feature_columns + added, added
ratio_name = f"{numerator}_over_{denominator}"
reverse_name = f"{denominator}_over_{numerator}"
if ratio_name in frame.columns or reverse_name in frame.columns:
continue
den = pd.to_numeric(frame[denominator], errors="coerce").replace(0, np.nan)
rev_den = pd.to_numeric(frame[numerator], errors="coerce").replace(0, np.nan)
frame[ratio_name] = pd.to_numeric(frame[numerator], errors="coerce") / den
frame[reverse_name] = pd.to_numeric(frame[denominator], errors="coerce") / rev_den
feature_components[ratio_name] = [numerator, denominator]
feature_components[reverse_name] = [denominator, numerator]
added.extend([ratio_name, reverse_name])
if len(added) >= max_ratio_features:
return feature_columns + added, added
return feature_columns + added, added
def _add_interaction_features(
frame: pd.DataFrame,
feature_columns: list[str],
feature_components: dict[str, list[str]],
max_interaction_features: int = 12,
) -> list[str]:
"""Multiplicative INTERACTION features z(a)*z(b) for the leading numeric pairs (S5).
Ratios capture one two-feature relationship; interactions capture another β an effect present only
when two features are jointly high/low, with neither main effect alone. Standardized/centered so the
product is a genuine interaction term (both-high & both-low β positive), not scale-dominated. Has its
OWN budget so it is never crowded out by the ratio budget. Detects BETWEEN-unit interactions; pure
within-person/within-row interactions still need a mixed-effects interaction term (documented).
"""
if max_interaction_features <= 0:
return []
# Prioritise the PRIMARY signal carrier of each distinct base feature for pairing: the central-
# tendency (`_mean`) aggregate, or the raw feature for cross-sectional data. Pairing the means of
# DISTINCT base features (activity_index_mean Γ sleep_index_mean) yields the interpretable, likely
# interactions; pairing a feature with its own std/min/max is redundant. This makes the meaningful
# interaction reachable within the small budget instead of being crowded out by same-family products.
def _base_of(col: str) -> str:
for suf in ("_mean", "_std", "_min", "_max", "_median", "_cv", "_trendslope",
"_weekend_minus_weekday", "_night_over_day"):
if col.endswith(suf):
return col[: -len(suf)]
return col
# Derive interaction candidates from the FULL frame, not the (already univariate-pruned)
# feature_columns list: an interaction-only biomarker has null MAIN effects, so its components
# are exactly the ones the univariate screen drops β they must still be reachable here.
frame_cols = [str(c) for c in frame.columns]
mean_cols = [c for c in frame_cols if c.endswith("_mean") and "_over_" not in c and "_x_" not in c]
primary = mean_cols if mean_cols else [c for c in feature_columns if "_over_" not in c and "_x_" not in c]
usable, seen_base = [], set()
for column in primary:
if "_over_" in column or "_x_" in column: # skip already-derived ratio/interaction features
continue
base = _base_of(column)
if base in seen_base:
continue
values = pd.to_numeric(frame[column], errors="coerce")
finite = values[np.isfinite(values)]
if len(finite) < 20 or finite.nunique() <= 2:
continue
usable.append(column)
seen_base.add(base)
usable = usable[:8] # cap pair explosion: 8 distinct base features -> up to 28 pairs, budget-capped
added: list[str] = []
for i, a in enumerate(usable):
for b in usable[i + 1:]:
if len(added) >= max_interaction_features:
return added
inter_name = f"{a}_x_{b}"
if inter_name in frame.columns:
continue
av = pd.to_numeric(frame[a], errors="coerce")
bv = pd.to_numeric(frame[b], errors="coerce")
sa = av.std(ddof=0)
sb = bv.std(ddof=0)
if not (sa and sb and np.isfinite(sa) and np.isfinite(sb)):
continue
prod = ((av - av.mean()) / sa) * ((bv - bv.mean()) / sb)
if prod.notna().sum() < 20 or prod.nunique(dropna=True) <= 2:
continue
frame[inter_name] = prod
feature_components[inter_name] = [a, b]
added.append(inter_name)
return added
def build_analysis_frame(
df: pd.DataFrame,
target_column: str,
participant_id_column: str | None = None,
time_column: str | None = None,
excluded_columns: list[str] | None = None,
confounder_columns: list[str] | None = None,
max_ratio_features: int = 24,
) -> AnalysisFrame:
if target_column not in df.columns:
raise InsufficientDataError(f"Target column '{target_column}' is not present in the dataset.")
# Constant target = nothing to associate/predict. (Too-few-observations is left
# to the downstream screening gate, which fires AFTER participant aggregation so
# legitimate small longitudinal cohorts are not blocked pre-aggregation.)
_target_numeric = pd.to_numeric(df[target_column], errors="coerce").dropna()
if len(_target_numeric) > 0 and float(_target_numeric.std(ddof=0) or 0.0) == 0.0:
raise InsufficientDataError(
f"Target '{target_column}' has no variance (all observed values are identical); there is nothing to associate or predict."
)
# Reject a target that is mostly non-numeric AND high-cardinality (a date,
# identifier, or free-text column mistakenly chosen as the outcome). A binary
# or low-cardinality categorical target is allowed and handled downstream.
# NOTE: compute the numeric fraction over OBSERVED (non-null) values so a
# heavily-missing but genuinely numeric target is not mistaken for text.
_observed_target = df[target_column].dropna()
_binary_target_note: str | None = None
if len(_observed_target) > 0:
_numeric_frac = float(pd.to_numeric(_observed_target, errors="coerce").notna().mean())
_target_unique = int(_observed_target.nunique())
if _numeric_frac < 0.5:
# Non-numeric (text/categorical) target. A 2-class target is encoded to 0/1 so the
# numeric screening/modeling can proceed (classification-style). High-cardinality looks
# like a date/id/free-text column, and 3-10 unordered text categories (e.g. a multi-level
# "sex" field) are not a usable single outcome β both are rejected with a clear message.
# Without this, a low-cardinality text target slipped through and crashed the numeric
# screening downstream (safe_spearman: "could not convert string to float: 'Male'").
if _target_unique > 10:
raise InsufficientDataError(
f"Target '{target_column}' is not a usable outcome: its observed values are mostly non-numeric with "
f"{_target_unique} distinct values, which looks like a date, identifier, or free-text column rather than a "
"clinical/behavioral endpoint. Choose a numeric or binary target."
)
if _target_unique == 2:
_cats = list(pd.Categorical(_observed_target).categories)
df = df.copy()
_codes = pd.Series(pd.Categorical(df[target_column], categories=_cats).codes, index=df.index)
df[target_column] = _codes.where(_codes >= 0)
_binary_target_note = (
f"Encoded binary categorical target '{target_column}' as 0/1 ({_cats[0]}=0, {_cats[1]}=1)."
)
else:
raise InsufficientDataError(
f"Target '{target_column}' is a non-numeric categorical column with {_target_unique} categories; "
"biomarker discovery requires a numeric or binary outcome. Choose a numeric endpoint, or a binary "
"target for classification."
)
excluded = set(excluded_columns or [])
confounders = [column for column in (confounder_columns or []) if column in df.columns]
audit_log: list[str] = []
if _binary_target_note:
audit_log.append(_binary_target_note)
# Outcome/label columns OTHER than the target are construct-circular as biomarker candidates
# (predicting one outcome from another). Block them up front and record the reason β this is the
# NAME-based complement to the value-based construct-validity hard gate, which only catches
# near-perfect proxies; a moderately-correlated alternative scale (e.g. GLOBEM target_BDI2 vs
# target_depression_weekly, rho~0.6) would slip the value gate yet is still circular. Recorded in
# excluded_columns + audit_log so the block is transparent (not a silent drop), and the column is
# retained in the frame for reference rather than promoted as a "discovered biomarker".
_blocked_outcomes = [
column for column in df.columns
if column != target_column and column not in confounders and column not in excluded
and _is_outcome_label_column(column)
]
if _blocked_outcomes:
excluded.update(_blocked_outcomes)
audit_log.append(
"Blocked outcome/label column(s) from biomarker candidates (using one outcome to predict "
f"another is construct-circular): {', '.join(map(str, _blocked_outcomes))}."
)
# Construct-overlap guard: when the target is a psychometric/self-report instrument, sibling
# self-report "*_score" survey columns are criterion-contaminated (self-report predicting
# self-report), not wearable biomarkers β exclude them (wearable device scores are allow-listed).
_blocked_overlap = [
column for column in df.columns
if column != target_column and column not in confounders and column not in excluded
and _is_construct_overlap_score(column, target_column)
]
if _blocked_overlap:
excluded.update(_blocked_overlap)
audit_log.append(
"Blocked self-report '*_score' survey column(s) as construct overlap with the self-report "
f"target '{target_column}' (self-report predicting self-report is circular): "
f"{', '.join(map(str, _blocked_overlap))}."
)
work = df.copy()
# Aggregate to one row per participant when a time axis is present OR when a
# participant appears in multiple rows (repeated measures). The latter guards
# against pseudo-replication: treating non-independent rows from the same
# person as independent observations would inflate effective n and significance.
_has_duplicate_participants = (
participant_id_column
and participant_id_column in work.columns
and int(work[participant_id_column].nunique(dropna=True)) < len(work)
)
# Aggregation is correct when each participant contributes repeated measures of a
# participant-LEVEL outcome (one HOMA-IR per person, a constant diagnosis). But when
# the TARGET itself varies WITHIN a participant (a stress label that is 0 at baseline
# and 1 under stress; a per-visit severity score) and there is no time axis to model
# it longitudinally, collapsing to one row per participant destroys the very signal
# we want β the target would aggregate to a near-constant. In that case keep the
# row-level data; within-participant correlation is handled by the mixed-effects model.
_target_varies_within_participant = False
if _has_duplicate_participants:
_target_for_variation = pd.to_numeric(work[target_column], errors="coerce")
_unique_per_participant = (
work.assign(_codas_target_var=_target_for_variation)
.groupby(participant_id_column)["_codas_target_var"]
.nunique(dropna=True)
)
if len(_unique_per_participant):
_target_varies_within_participant = float((_unique_per_participant >= 2).mean()) >= 0.20
_should_aggregate = bool(
participant_id_column
and participant_id_column in work.columns
and (time_column or _has_duplicate_participants)
)
if _should_aggregate and not time_column and _target_varies_within_participant:
_should_aggregate = False
audit_log.append(
f"Kept row-level repeated measures: target '{target_column}' varies within "
f"'{participant_id_column}' for a substantial fraction of participants, so participant "
"aggregation would erase the within-person signal. Within-participant correlation is "
"addressed by the mixed-effects model instead."
)
if _should_aggregate:
work = _aggregate_longitudinal(work, target_column, participant_id_column, time_column, excluded, confounders)
target_note = work.attrs.get("codas_target_aggregation_note", "target aggregation")
if time_column:
audit_log.append(
f"Aggregated longitudinal rows by '{participant_id_column}' using mean/std/min/max/median features and {target_note}."
)
else:
audit_log.append(
f"Aggregated repeated-measures rows (no time column detected) by '{participant_id_column}' to avoid "
f"pseudo-replication, using mean/std/min/max/median features and {target_note}."
)
elif participant_id_column and participant_id_column not in work.columns:
audit_log.append(f"Participant ID column '{participant_id_column}' was not found and was ignored.")
participant_id_column = None
reserved = {target_column}
if participant_id_column:
reserved.add(participant_id_column)
if time_column:
reserved.add(time_column)
reserved.update(excluded)
reserved.update(confounders)
feature_columns = [
column
for column in work.columns
if column not in reserved
and not _is_demographic_column(column) # demographics/SES are confounders, not biomarker candidates
and not _is_outcome_label_column(column) # other outcome/label columns are circular, not candidates
and pd.api.types.is_numeric_dtype(work[column])
]
feature_columns = [
column
for column in feature_columns
if work[column].notna().sum() >= 20 and work[column].nunique(dropna=True) > 1
]
# Drop trivial identifier / row-index columns that survived as numeric features. A column whose
# NAME looks like an id/index AND whose values are ~unique per row is a record id, not a
# biomarker. Without this, a sequential id that happens to track the target (sorted exports β
# the A15 trivial-leakage trap) could be screened and ratio-engineered into a spurious
# "biomarker". Requiring BOTH the name pattern AND near-uniqueness keeps real measurements
# (e.g. a low-cardinality "device_id" grouping, or any genuinely continuous feature).
_ID_NAMES = {
"id", "index", "rowid", "row_id", "row_index", "rowindex", "record_id", "recordid",
"record_number", "uuid", "guid", "seq", "sequence", "sample_id", "sampleid", "row",
"row_number", "rownum", "serial", "serial_number",
}
_AGG_SUFFIXES = ("_mean", "_median", "_min", "_max", "_std", "_sum", "_count", "_first", "_last", "_iqr", "_range")
def _is_identifier_like(col: str) -> bool:
# STRUCTURAL row-index/ID detector (name-independent): a column whose values are all-unique
# CONSECUTIVE integers (an arange-like 0..n-1 / 1..n in any order) is a row index or record id,
# never a continuous biomarker. Caught regardless of column name β fixes the row-order leakage
# where a sequential id on a target-SORTED export was screened+validated as a "biomarker"
# (found via injection probe on thyroid_recurrence / obesity_level, which are sorted by target).
_series0 = work[col].dropna()
_n0 = len(_series0)
if _n0 >= 20 and _series0.nunique() == _n0:
_vals = pd.to_numeric(_series0, errors="coerce").dropna()
if len(_vals) == _n0 and np.all(np.isfinite(_vals)) and np.allclose(np.mod(_vals, 1.0), 0.0):
_span = float(_vals.max() - _vals.min())
if abs(_span - (_n0 - 1)) <= max(1.0, 0.001 * _n0): # consecutive integer sequence
return True
name = str(col).strip().lower()
# Longitudinal aggregation renames "record_id" -> "record_id_mean"/"_min"/... so match the
# base name (suffix stripped) as well as the raw name.
base = name
for suffix in _AGG_SUFFIXES:
if base.endswith(suffix):
base = base[: -len(suffix)]
break
name_hit = any(
candidate in _ID_NAMES
or candidate.endswith("_id") or candidate.endswith("_index")
or candidate.endswith("_uuid") or candidate.endswith("_guid")
or candidate.startswith("unnamed:")
for candidate in (name, base)
)
if not name_hit:
return False
series = work[col].dropna()
return len(series) >= 20 and series.nunique() >= 0.95 * len(series)
identifier_like = [column for column in feature_columns if _is_identifier_like(column)]
if identifier_like:
feature_columns = [column for column in feature_columns if column not in identifier_like]
audit_log.append(
f"Excluded identifier/index-like columns from biomarker features (near-unique per row): "
f"{', '.join(map(str, identifier_like))}."
)
feature_components = {column: [column] for column in feature_columns}
feature_columns, added_ratios = _add_ratio_features(
work,
feature_columns,
feature_components,
max_ratio_features=max_ratio_features,
)
if added_ratios:
audit_log.append(f"Generated {len(added_ratios)} ratio features before label-aware screening.")
# Interaction (product) features get their OWN budget so they are never crowded out by ratios.
try:
added_interactions = _add_interaction_features(work, feature_columns, feature_components)
if added_interactions:
feature_columns = feature_columns + added_interactions
audit_log.append(f"Generated {len(added_interactions)} interaction (product) features before screening.")
except Exception:
pass # interactions are additive; never fail the frame build
keep_columns = list(dict.fromkeys(
[column for column in [participant_id_column, target_column] if column]
+ confounders
+ feature_columns
+ [column for column in excluded if column in work.columns]
))
work = work[keep_columns].replace([np.inf, -np.inf], np.nan)
return AnalysisFrame(
frame=work,
target_column=target_column,
participant_id_column=participant_id_column,
time_column=time_column,
feature_columns=feature_columns,
confounder_columns=confounders,
excluded_columns=[column for column in excluded if column in work.columns],
feature_components=feature_components,
audit_log=audit_log,
)
|