File size: 28,499 Bytes
12f12d7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
"""Implementation of the CoDaS internal validation battery."""

from __future__ import annotations

import os
from dataclasses import dataclass

import numpy as np
import pandas as pd
from scipy import stats as _stats
from sklearn.model_selection import train_test_split

from .models import BiomarkerCandidate, ValidationTestResult
from .statistics import (
    auc_against_target,
    partial_spearman,
    safe_kendall,
    safe_pearson,
    safe_spearman,
    same_sign,
    spearman_rho_fast,
)


def _near_deterministic_of_target(x, y, threshold: float = 0.97) -> bool:
    """Does a SINGLE feature near-perfectly determine a low-cardinality (categorical) target?
    Bins the feature into quantiles and computes majority-vote class accuracy; ~1.0 means the feature
    is a copy/proxy of the outcome (leakage), regardless of Spearman rho. Generalises the binary-AUC
    ceiling to MULTICLASS targets (where Spearman saturates and binary AUC is undefined) — e.g. a
    near-perfect leak of a 3-class fetal-health target reads Spearman ~0.72 yet determines the class."""
    xv = pd.to_numeric(pd.Series(x), errors="coerce")
    yv = pd.Series(y).reset_index(drop=True)
    xv = xv.reset_index(drop=True)
    mask = xv.notna() & yv.notna()
    xv, yv = xv[mask], yv[mask]
    n = int(len(xv))
    k = int(yv.nunique())
    if n < 30 or k < 2 or k > 20:  # only categorical / low-cardinality targets
        return False
    # Enough bins to RESOLVE class boundaries: on an imbalanced multiclass target a coarse binning
    # leaves boundary bins mixed and caps a true separator's purity ~0.96. Fine bins push a real
    # separator to ~0.99 while a legitimate strong feature stays ~0.80-0.85 (clean margin).
    nb = min(100, max(40, n // 25), n)
    try:
        bins = pd.qcut(xv.rank(method="first"), q=nb, duplicates="drop")
    except Exception:
        return False
    grp = pd.DataFrame({"b": bins.values, "y": yv.values}).groupby("b", observed=True)["y"]
    correct = int(grp.apply(lambda s: int(s.value_counts().iloc[0])).sum())
    return (correct / n) >= threshold


def _cyclic_confounder_period(name: str) -> float | None:
    """Period (raw units) for a high-cardinality CYCLIC time confounder, so a circadian/seasonal
    rhythm can be adjusted with Fourier (sin/cos) terms. Hour-of-day is cyclic (0 ≈ 23) and its
    effect is sinusoidal: a linear adjustment leaves the circadian confound intact, which would
    let a time-of-day rhythm masquerade as a stress/physiology biomarker. Low-cardinality cyclics
    (month, weekday) fall through to one-hot, which is more flexible within the cap."""
    low = str(name).lower()
    if "hour" in low:
        return 24.0
    if "minute" in low:
        return 60.0
    if "dayofyear" in low or "day_of_year" in low or low == "doy":
        return 365.0
    return None


def _is_adjustable_confounder(series: pd.Series) -> bool:
    """A confounder can be adjusted for if it is numeric, OR a low-cardinality categorical —
    including a STRING-coded one (a real multi-site study codes site as 'BWH'/'MGH', not 0/1/2).
    High-cardinality strings (free-text, IDs) are not adjustable and are skipped."""
    if pd.api.types.is_numeric_dtype(series):
        return True
    return 2 <= int(series.dropna().nunique()) <= 12


def _confounder_covariate_matrix(frame: pd.DataFrame, confounders: list[str], index) -> np.ndarray:
    """Build the covariate matrix for partial-correlation confounder adjustment.

    * CYCLIC time confounders (hour/minute/day-of-year) are expanded into sin/cos (Fourier)
      terms so a circadian/seasonal rhythm is actually removed (linear/one-hot can't capture a
      cyclic, high-cardinality confound — this is the classic wearable "it's just time-of-day").
    * Multi-level categorical confounders (a site/batch code, 3-12 discrete levels) are one-hot
      encoded so a category-specific (step) confound is fully adjusted, not under-adjusted by a
      single linear term.
    * Binary (sex, group) and continuous (age, bmi) confounders stay a single column -- unchanged.
    NaN rows are preserved as NaN so partial_spearman drops incomplete observations.
    """
    blocks: list[np.ndarray] = []
    sub = frame.loc[index, confounders]
    n_rows = len(sub)
    for col in confounders:
        raw = sub[col]
        # STRING / non-numeric categorical confounder (e.g. site "BWH"/"MGH") -> one-hot.
        if not pd.api.types.is_numeric_dtype(raw):
            nun = int(raw.dropna().nunique())
            if 2 <= nun <= 12:
                dummies = pd.get_dummies(raw.astype("object"), prefix=col, drop_first=True, dtype=float)
                arr = np.array(dummies.to_numpy(dtype=float))  # writable copy (pandas>=3.0 CoW returns a read-only view)
                if arr.size:
                    arr[raw.isna().to_numpy(), :] = np.nan
                    blocks.append(arr)
                    continue
            continue  # high-cardinality / unusable string contributes nothing
        s = pd.to_numeric(raw, errors="coerce")
        non_na = s.dropna()
        if len(non_na) == 0:
            blocks.append(s.to_numpy(dtype=float).reshape(-1, 1))
            continue
        period = _cyclic_confounder_period(col)
        if period is not None:
            v = s.to_numpy(dtype=float)
            block = np.column_stack([np.sin(2 * np.pi * v / period), np.cos(2 * np.pi * v / period)])
            block[s.isna().to_numpy(), :] = np.nan
            blocks.append(block)
            continue
        nunique = int(non_na.nunique())
        is_integer_valued = bool(np.all(np.mod(non_na.to_numpy(dtype=float), 1.0) == 0.0))
        if 3 <= nunique <= 12 and is_integer_valued:
            dummies = pd.get_dummies(s, prefix=col, drop_first=True, dtype=float)
            arr = np.array(dummies.to_numpy(dtype=float))  # writable copy (pandas>=3.0 CoW returns a read-only view)
            if arr.size:
                arr[s.isna().to_numpy(), :] = np.nan  # let partial_spearman mask incomplete rows
                blocks.append(arr)
                continue
        blocks.append(s.to_numpy(dtype=float).reshape(-1, 1))
    if not blocks:
        return np.empty((n_rows, 0), dtype=float)
    return np.column_stack(blocks)


@dataclass
class ValidationConfig:
    alpha: float = 0.05
    n_resamples: int = 1000
    random_state: int = 17
    construct_threshold: float = 0.85
    min_holdout_n: int = 20
    # Leave-one-out influence is a sanity check for single-point leverage; 200 randomly
    # sampled points detect influential observations adequately and cost ~10x less than
    # 2000 (which dominated runtime on large datasets). Override via CODAS_MAX_LOO_CHECKS.
    max_loo_checks: int = int(os.getenv("CODAS_MAX_LOO_CHECKS", "200"))
    fdr_alpha: float = 0.10


def _valid_arrays(frame: pd.DataFrame, feature: str, target: str) -> tuple[np.ndarray, np.ndarray, pd.DataFrame]:
    subset = frame[[feature, target]].replace([np.inf, -np.inf], np.nan).dropna()
    return subset[feature].to_numpy(dtype=float), subset[target].to_numpy(dtype=float), subset


def _split_holdout(
    frame: pd.DataFrame,
    feature: str,
    target: str,
    participant_id_column: str | None,
    random_state: int,
) -> pd.DataFrame:
    needed = [feature, target]
    if participant_id_column and participant_id_column in frame.columns:
        needed.append(participant_id_column)
    subset = frame[needed].replace([np.inf, -np.inf], np.nan).dropna()
    if participant_id_column and participant_id_column in subset.columns:
        # np.asarray: a pandas string/Arrow-backed participant column yields an extension array that
        # sklearn cannot integer-index; coerce to a plain numpy array so train_test_split works.
        ids = np.asarray(subset[participant_id_column].dropna().unique())
        if len(ids) < 5:
            return subset.iloc[0:0]
        _, test_ids = train_test_split(ids, test_size=0.2, random_state=random_state)
        return subset[subset[participant_id_column].isin(list(test_ids))]
    if len(subset) < 25:
        return subset.iloc[0:0]
    _, test = train_test_split(subset, test_size=0.2, random_state=random_state)
    return test


def _bootstrap_distribution(
    x: np.ndarray,
    y: np.ndarray,
    n_resamples: int,
    random_state: int,
) -> np.ndarray:
    # Same indices (same RNG draws) as before; only the per-resample rho is computed via the fast
    # rank+Pearson path (skips scipy's discarded p-value). x/y are pre-dropna'd (finite).
    rng = np.random.default_rng(random_state)
    values = []
    n = len(x)
    for _ in range(n_resamples):
        index = rng.integers(0, n, n)
        rho = spearman_rho_fast(x[index], y[index])
        if np.isfinite(rho):
            values.append(rho)
    return np.asarray(values, dtype=float)


def _permutation_p_value(
    x: np.ndarray,
    y: np.ndarray,
    observed_rho: float,
    n_resamples: int,
    random_state: int,
) -> float:
    # Spearman(x, permute(y)) == Pearson(rank(x), permute(rank(y))) exactly, and permuting the ranks
    # leaves their mean/std unchanged, so each null rho is a single dot product against precomputed,
    # mean-centred ranks — no per-resample re-ranking. rng.permutation(n) consumes the RNG identically
    # to the old rng.permutation(y), so the null distribution (and p-value) is bit-identical.
    rng = np.random.default_rng(random_state)
    n = len(x)
    if n < 3 or not np.isfinite(observed_rho):
        return float("nan")
    rx = _stats.rankdata(x)
    ry = _stats.rankdata(y)
    sx = rx.std()
    sy = ry.std()
    if sx < 1e-12 or sy < 1e-12:
        return float("nan")
    rxc = rx - rx.mean()
    ryc = ry - ry.mean()
    denom = n * sx * sy
    target = abs(observed_rho)
    ge = 0
    total = 0
    for _ in range(n_resamples):
        perm = rng.permutation(n)
        rho = float(np.dot(rxc, ryc[perm]) / denom)
        if np.isfinite(rho):
            total += 1
            if abs(rho) >= target:
                ge += 1
    if total == 0:
        return float("nan")
    return float((ge + 1) / (total + 1))


def validate_candidate(
    frame: pd.DataFrame,
    candidate: BiomarkerCandidate,
    target_column: str,
    participant_id_column: str | None,
    confounder_columns: list[str],
    excluded_columns: list[str],
    feature_components: dict[str, list[str]],
    config: ValidationConfig | None = None,
) -> BiomarkerCandidate:
    config = config or ValidationConfig()
    feature = candidate.feature
    x, y, subset = _valid_arrays(frame, feature, target_column)
    tests: list[ValidationTestResult] = []

    # NOTE: this is a within-sample 20% resplit of the SAME data used to screen/select the
    # candidate, NOT a pre-selection holdout, so it is an optimistic stability check rather than
    # true out-of-sample replication. Named `subsample_replication` (not `independent_replication`)
    # to avoid overclaiming. Selection multiplicity is controlled by the FDR screen, not this test.
    holdout = _split_holdout(frame, feature, target_column, participant_id_column, config.random_state)
    if len(holdout) >= config.min_holdout_n:
        holdout_rho, holdout_p, holdout_n = safe_spearman(holdout[feature], holdout[target_column])
        passed = same_sign(candidate.rho, holdout_rho) and holdout_p <= config.alpha
        tests.append(ValidationTestResult(
            name="subsample_replication",
            dimension="replication",
            passed=passed,
            metric=holdout_rho,
            p_value=holdout_p,
            details=f"within-sample 20% resplit (not a pre-selection holdout); holdout_n={holdout_n}",
        ))
    else:
        tests.append(ValidationTestResult(
            name="subsample_replication",
            dimension="replication",
            passed=False,
            applicable=False,
            details=f"holdout_n={len(holdout)} below minimum {config.min_holdout_n}",
        ))

    perm_p = _permutation_p_value(x, y, candidate.rho, config.n_resamples, config.random_state)
    tests.append(ValidationTestResult(
        name="permutation_test",
        dimension="replication",
        passed=np.isfinite(perm_p) and perm_p <= config.alpha,
        p_value=perm_p,
        details=f"resamples={config.n_resamples}",
    ))

    boot = _bootstrap_distribution(x, y, config.n_resamples, config.random_state)
    if len(boot) >= max(50, config.n_resamples // 10):
        ci_low, ci_high = np.percentile(boot, [2.5, 97.5])
        ci_mid = float((ci_low + ci_high) / 2)
        bootstrap_passed = (ci_low > 0 and ci_high > 0) or (ci_low < 0 and ci_high < 0)
        tests.append(ValidationTestResult(
            name="bootstrap_stability",
            dimension="stability",
            passed=bool(bootstrap_passed),
            metric=float(ci_mid),
            details=f"ci95=[{ci_low:.4g}, {ci_high:.4g}], resamples={len(boot)}",
        ))
    else:
        ci_low = ci_high = ci_mid = float("nan")
        tests.append(ValidationTestResult(
            name="bootstrap_stability",
            dimension="stability",
            passed=False,
            applicable=False,
            details="insufficient valid bootstrap resamples",
        ))

    loo_indices = np.arange(len(x))
    if len(loo_indices) > config.max_loo_checks:
        rng = np.random.default_rng(config.random_state)
        loo_indices = np.sort(rng.choice(loo_indices, config.max_loo_checks, replace=False))
    sign_flips = 0
    for idx in loo_indices:
        mask = np.ones(len(x), dtype=bool)
        mask[idx] = False
        rho, _, _ = safe_spearman(x[mask], y[mask])
        if not same_sign(candidate.rho, rho):
            sign_flips += 1
            break
    tests.append(ValidationTestResult(
        name="leave_one_out_influence",
        dimension="stability",
        passed=sign_flips == 0 and len(loo_indices) > 0,
        metric=float(sign_flips),
        details=f"checked={len(loo_indices)}",
    ))

    median = np.nanmedian(y)
    lower = subset[subset[target_column] <= median]
    upper = subset[subset[target_column] > median]
    if len(lower) >= 10 and len(upper) >= 10:
        lower_rho, lower_p, _ = safe_spearman(lower[feature], lower[target_column])
        upper_rho, upper_p, _ = safe_spearman(upper[feature], upper[target_column])
        passed = same_sign(candidate.rho, lower_rho) and same_sign(candidate.rho, upper_rho)
        tests.append(ValidationTestResult(
            name="subgroup_consistency",
            dimension="robustness",
            passed=passed,
            metric=float(min(abs(lower_rho), abs(upper_rho))),
            p_value=float(max(lower_p, upper_p)),
            details=f"median_split=({len(lower)}, {len(upper)})",
        ))
    else:
        tests.append(ValidationTestResult(
            name="subgroup_consistency",
            dimension="robustness",
            passed=False,
            applicable=False,
            details="not enough rows after median split",
        ))

    # Sequential (temporal / batch) split consistency: divide the rows in their GIVEN ORDER into
    # a first half and second half. If the association holds in the first half but NOT in the
    # second half (or vice versa), the effect may reflect a batch, drift, or cohort artefact
    # rather than a stable biological relationship. This is especially important in longitudinal
    # datasets where the first and second halves may correspond to different periods or cohorts.
    # NOT a hard gate (the first/last ordering may be arbitrary in cross-sectional data);
    # surfaces as an instability note so a reviewer can check whether their data is ordered by
    # time or cohort, in which case this inconsistency is a meaningful replication concern.
    n_sub = len(subset)
    if n_sub >= 40:
        first_half = subset.iloc[: n_sub // 2]
        second_half = subset.iloc[n_sub // 2 :]
        rho_f, _, _ = safe_spearman(first_half[feature], first_half[target_column])
        rho_s, _, _ = safe_spearman(second_half[feature], second_half[target_column])
        # Consistent = same sign in BOTH halves AND neither half is near-zero relative to the other.
        # A 10x magnitude drop (e.g. ρ=-0.56 vs ρ=-0.05) is a replication concern even if signs agree.
        _mag_ratio = (
            min(abs(rho_f), abs(rho_s)) / max(abs(rho_f), abs(rho_s))
            if max(abs(rho_f), abs(rho_s)) > 1e-9 else 1.0
        )
        consistent = (
            np.isfinite(rho_f) and np.isfinite(rho_s)
            and same_sign(candidate.rho, rho_f) and same_sign(candidate.rho, rho_s)
            and _mag_ratio >= 0.25  # second-half effect must be at least 25% of first-half
        )
        tests.append(ValidationTestResult(
            name="sequential_split_consistency",
            dimension="stability",
            passed=consistent,
            # Not a hard gate — only flags drift/batch concern; does not block validation
            applicable=True,
            metric=float(min(abs(rho_f), abs(rho_s))) if (np.isfinite(rho_f) and np.isfinite(rho_s)) else float("nan"),
            details=(
                f"first_half_rho={rho_f:.3f}, second_half_rho={rho_s:.3f} — "
                + ("UNSTABLE: different directions in first vs second half; check if rows are ordered "
                   "by time/cohort (if so, this is a replication concern)" if not consistent else "stable")
            ),
        ))

    pearson_r, pearson_p, _ = safe_pearson(x, y)
    kendall_tau, kendall_p, _ = safe_kendall(x, y)
    method_passed = (
        same_sign(candidate.rho, pearson_r)
        and same_sign(candidate.rho, kendall_tau)
        and pearson_p <= config.alpha
        and kendall_p <= config.alpha
    )
    tests.append(ValidationTestResult(
        name="method_triangulation",
        dimension="robustness",
        passed=method_passed,
        metric=float(min(abs(pearson_r), abs(kendall_tau))),
        p_value=float(max(pearson_p, kendall_p)),
        details=f"pearson={pearson_r:.4g}, kendall_tau={kendall_tau:.4g}",
    ))

    # Construct-validity hard gate: a feature so target-like that it is a copy/proxy of the outcome
    # (leakage), not an independent biomarker. Spearman |rho|>threshold catches near-perfect CONTINUOUS
    # leaks, but Spearman SATURATES on a binary target (a perfect class separator reads rho~0.8, below
    # the 0.85 threshold), so a near-deterministic copy of a binary outcome would slip through. Add a
    # class-separation ceiling: an essentially-perfect single-feature separator (AUC>=0.99 either
    # direction) is leakage regardless of Spearman rho. (Found via adversarial leak-injection across 12
    # real datasets: an injected target-copy was validated on 7 binary-target datasets.)
    construct_auc, _construct_auc_n = auc_against_target(y, x)
    near_perfect_separator = (
        (np.isfinite(construct_auc) and max(construct_auc, 1.0 - construct_auc) >= 0.99)
        or _near_deterministic_of_target(x, y)  # multiclass-aware (binary AUC undefined for >2 classes)
    )
    construct_passed = not (
        candidate.n > 30
        and (
            (np.isfinite(candidate.rho) and abs(candidate.rho) > config.construct_threshold)
            or near_perfect_separator
        )
    )
    tests.append(ValidationTestResult(
        name="construct_validity_hard_gate",
        dimension="robustness",
        passed=construct_passed,
        hard_gate=True,
        metric=abs(candidate.rho),
        details=f"threshold={config.construct_threshold}, auc={construct_auc:.4g} (near_perfect_separator={near_perfect_separator})",
    ))

    usable_confounders = [
        column
        for column in confounder_columns
        if column in frame.columns and _is_adjustable_confounder(frame[column])
    ]
    if usable_confounders:
        covariates = _confounder_covariate_matrix(frame, usable_confounders, subset.index)
        partial_rho, partial_p, partial_n = partial_spearman(x, y, covariates)
        confounder_adjusted_passed = same_sign(candidate.rho, partial_rho) and partial_p <= config.alpha
        tests.append(ValidationTestResult(
            name="confounder_adjusted_robustness",
            dimension="robustness",
            passed=confounder_adjusted_passed,
            metric=partial_rho,
            p_value=partial_p,
            details=f"partial_spearman_n={partial_n}, covariates={usable_confounders}",
        ))
        # HARD GATE: if the association does not SURVIVE adjustment for the supplied
        # confounders, the feature is confounded, not an independent biomarker, and must
        # not be promoted to "validated". Three independent criteria for confounding:
        #   (a) Sign flip after adjustment — the "biomarker" reverses direction.
        #   (b) Effect collapses to <25% of the raw effect (magnitude attenuation ≥75%).
        #   (c) Partial effect is statistically non-significant (p>α) while still having a
        #       substantial raw effect (|raw_rho|>0.25). This is the key case for masking
        #       confounded associations: the partial correlation may retain the correct sign
        #       but be explicable entirely by the confounder.
        # Low-power samples (n<50) are exempted from (c) to avoid false downgrades when the
        # partial test simply lacks power.
        severe_sign_flip = (
            np.isfinite(partial_rho) and np.isfinite(candidate.rho)
            and abs(candidate.rho) > 1e-9
            and not same_sign(candidate.rho, partial_rho)
        )
        severe_collapse = (
            np.isfinite(partial_rho) and np.isfinite(candidate.rho)
            and abs(candidate.rho) > 1e-9
            and abs(partial_rho) < 0.25 * abs(candidate.rho)
        )
        severe_nonsig = (
            partial_n >= 50
            and np.isfinite(partial_rho) and np.isfinite(candidate.rho)
            and abs(candidate.rho) > 0.25           # meaningful raw effect
            and partial_p > config.alpha             # but partial is non-significant
        )
        severely_confounded = partial_n >= 50 and (severe_sign_flip or severe_collapse or severe_nonsig)
        if severely_confounded:
            _reason = (
                "sign flip after confounder adjustment" if severe_sign_flip
                else f"effect magnitude collapses to {abs(partial_rho):.3f} (raw={abs(candidate.rho):.3f}, attenuation≥75%)" if severe_collapse
                else f"partial effect non-significant after adjustment (partial_p={partial_p:.3g}>α={config.alpha}, partial_rho={partial_rho:.3f})"
            )
            _conf_msg = (
                f"raw_rho={candidate.rho:.3f}, partial_rho={partial_rho:.3f} given {usable_confounders}: "
                f"confounded — {_reason}. "
                f"The raw association appears to be driven by {usable_confounders} rather than an "
                f"independent relationship with the target."
            )
        else:
            _conf_msg = (
                f"raw_rho={candidate.rho:.3f}, partial_rho={partial_rho:.3f} given {usable_confounders}: "
                f"survives confounder adjustment"
            )
        tests.append(ValidationTestResult(
            name="confounder_independence_hard_gate",
            dimension="robustness",
            passed=not severely_confounded,
            hard_gate=True,
            applicable=True,
            metric=partial_rho,
            p_value=partial_p,
            details=_conf_msg,
        ))
    else:
        tests.append(ValidationTestResult(
            name="confounder_adjusted_robustness",
            dimension="robustness",
            passed=False,
            applicable=False,
            details="no numeric confounders supplied",
        ))

    independence_failures = []
    components = feature_components.get(feature, [feature])
    for column in excluded_columns:
        if column in frame.columns and pd.api.types.is_numeric_dtype(frame[column]):
            rho, _, _ = safe_spearman(frame[feature], frame[column])
            if np.isfinite(rho) and abs(rho) > config.construct_threshold:
                independence_failures.append(f"excluded:{column}")
    for component in components:
        if component != feature and component in frame.columns:
            rho, _, _ = safe_spearman(frame[component], frame[target_column])
            if np.isfinite(rho) and abs(rho) > config.construct_threshold:
                independence_failures.append(f"component_proxy:{component}")
    tests.append(ValidationTestResult(
        name="construct_independence_hard_gate",
        dimension="robustness",
        passed=not independence_failures,
        hard_gate=True,
        details=", ".join(independence_failures) if independence_failures else "independent",
    ))

    ci_available = bool(np.isfinite(ci_mid))
    ci_consistent = ci_available and same_sign(candidate.rho, ci_mid)
    tests.append(ValidationTestResult(
        name="ci_consistency_hard_gate",
        dimension="stability",
        passed=ci_consistent,
        applicable=ci_available,
        hard_gate=True,
        metric=ci_mid,
        details=(
            "bootstrap CI midpoint sign compared with point estimate"
            if ci_available
            else "not applied because bootstrap CI could not be estimated"
        ),
    ))

    auc, auc_n = auc_against_target(y, x)
    tests.append(ValidationTestResult(
        name="discriminative_power",
        dimension="discriminative_power",
        passed=np.isfinite(auc) and auc >= 0.55,
        metric=auc,
        details=f"auc_n={auc_n}, threshold=0.55",
    ))

    # Multiple-comparison discipline: a candidate that did not survive the
    # Benjamini-Hochberg FDR screen must never be promoted to "validated", no
    # matter how many per-feature resampling checks it passes by chance. This is
    # the single guard that keeps the agent from "discovering" biomarkers in
    # noise when the FDR screen correctly found nothing significant.
    fdr_significant = bool(np.isfinite(candidate.q_value) and candidate.q_value <= config.fdr_alpha)
    tests.append(ValidationTestResult(
        name="multiple_comparison_screen",
        dimension="multiplicity",
        passed=fdr_significant,
        metric=float(candidate.q_value) if np.isfinite(candidate.q_value) else None,
        details=f"benjamini_hochberg_q={candidate.q_value:.4g}, fdr_alpha={config.fdr_alpha}",
    ))

    applicable = [test for test in tests if test.applicable]
    passed_count = sum(1 for test in applicable if test.passed)
    pass_rate = passed_count / len(applicable) if applicable else 0.0
    test_map = {test.name: test for test in tests}
    core_names = {
        "subsample_replication",
        "permutation_test",
        "bootstrap_stability",
        "ci_consistency_hard_gate",
    }
    core_passed = all(test_map[name].passed for name in core_names if test_map[name].applicable)
    hard_gate_failed = any(test.hard_gate and test.applicable and not test.passed for test in tests)
    first_three_failed = all(
        (not test_map[name].passed)
        for name in ("subsample_replication", "permutation_test", "bootstrap_stability")
        if test_map[name].applicable
    )

    if hard_gate_failed or first_three_failed:
        verdict = "rejected"
    elif pass_rate >= 0.70 and core_passed and fdr_significant:
        verdict = "validated"
    elif pass_rate >= 0.40:
        # NOTE: intentionally NOT gated on fdr_significant. On a true-null dataset the
        # acceptance harness relies on the top candidates being carried forward as
        # "conditional" so the negative-control (label-shuffle permutation null) and
        # confounder-attenuation analyses can run *on them* and demonstrate the null.
        # Emptying this bucket breaks that null-demonstration pipeline. The "this is not
        # a finding" honesty is enforced at the synthesis layer instead (explicit
        # "0 passed the full validation battery" headline + per-signal verdict labels).
        verdict = "conditional"
    else:
        verdict = "rejected"

    candidate.tests = tests
    candidate.pass_rate = float(pass_rate)
    candidate.verdict = verdict
    candidate.components = components
    candidate.score = float(abs(candidate.rho) * (1.0 - min(candidate.q_value, 1.0)) * (0.5 + pass_rate))
    candidate.evidence = f"{passed_count}/{len(applicable)} applicable validation checks passed."
    return candidate