File size: 4,936 Bytes
04ae9bb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
"""Patient-aware protein/UPDRS association statistics, not feature selection."""
from __future__ import annotations

import numpy as np
from scipy.stats import rankdata, t


def normalized_ranks(x, axis=0):
    ranks = rankdata(x, axis=axis, method="average")
    centered = ranks-ranks.mean(axis=axis, keepdims=True)
    norm = np.sqrt((centered**2).sum(axis=axis, keepdims=True))
    return np.divide(centered, norm, out=np.zeros_like(centered), where=norm > 0)


def permutation_spearman_columns(x, y, permutations=99999, seed=42):
    """Complete cases with identical missingness; permutation unit is patient.

    Two-sided absolute-rho test of independence, conditional on the observed
    paired cases. Ties use average ranks, randomized p-values use +1 correction.
    """
    x = np.asarray(x, dtype=float)
    if x.ndim == 1:
        x = x[:, None]
    a, b = normalized_ranks(x), normalized_ranks(np.asarray(y, dtype=float))
    observed = a.T @ b
    exceed = np.zeros(x.shape[1], dtype=np.int64)
    rng = np.random.default_rng(seed)
    for start in range(0, permutations, 2048):
        count = min(2048, permutations-start)
        shuffled = rng.permuted(np.broadcast_to(b, (count, len(b))), axis=1)
        null = shuffled @ a
        exceed += (np.abs(null) >= np.abs(observed)[None, :]-1e-12).sum(axis=0)
    return observed, (exceed+1)/(permutations+1)


def covariates(month, detected):
    time = rankdata(month).astype(float)
    time = (time-time.mean()) / max(time.std(), 1.)
    count = rankdata(detected).astype(float)
    count = (count-count.mean()) / max(count.std(), 1.)
    return np.c_[np.ones(len(time)), time, time**2, count]


def orthogonal_basis(z):
    u, s, _ = np.linalg.svd(z, full_matrices=False)
    if not len(s) or s[0] == 0:
        return np.empty((len(z), 0))
    rank = int((s > s[0]*max(z.shape)*np.finfo(float).eps).sum())
    return u[:, :rank]


def residualize(values, q):
    return values-q @ (q.T @ values)


def adjusted_rank_association(x, y, month, detected):
    """Partial rank correlation; approximate HC3-robust t test for rank slope."""
    x, y = rankdata(x).astype(float), rankdata(y).astype(float)
    q = orthogonal_basis(covariates(month, detected))
    xr, yr = residualize(x, q), residualize(y, q)
    xx, yy = xr @ xr, yr @ yr
    if xx < 1e-12 or yy < 1e-12:
        return np.nan, np.nan
    beta = xr @ yr / xx
    rho = (xr @ yr)/np.sqrt(xx*yy)
    error = yr-beta*xr
    leverage = (q**2).sum(1)+xr**2/xx
    if np.any(leverage >= 1-1e-10):
        return float(rho), np.nan
    variance = np.sum((xr*error/(1-leverage))**2)/xx**2
    pvalue = 2*t.sf(abs(beta)/np.sqrt(max(variance, 1e-30)), len(x)-q.shape[1]-1)
    return float(rho), float(pvalue)


def group_center(x, groups):
    x = np.asarray(x, dtype=float)
    vector = x.ndim == 1
    values = x[:, None] if vector else x
    size = np.bincount(groups)
    sums = np.zeros((len(size), values.shape[1]))
    np.add.at(sums, groups, values)
    result = values-sums[groups]/size[groups, None]
    return result[:, 0] if vector else result


def within_patient_association(x, y, month, detected, patients):
    """Rank association after patient FE + time/detection adjustment.

    Cluster-robust CR1 t test by patient; descriptive rank partial correlation.
    This is not the ordinary independent-observation Spearman test.
    """
    _, groups = np.unique(patients, return_inverse=True)
    keep = np.bincount(groups)[groups] >= 2
    x, y, month, detected, patients = [np.asarray(v)[keep] for v in [x, y, month, detected, patients]]
    _, groups = np.unique(patients, return_inverse=True)
    n, g = len(x), len(np.unique(groups))
    if g < 20 or n < 50:
        return np.nan, np.nan, n, g
    xw, yw = [group_center(rankdata(v), groups) for v in [x, y]]
    zw = group_center(covariates(month, detected)[:, 1:], groups)
    q = orthogonal_basis(zw)
    xr, yr = residualize(xw, q), residualize(yw, q)
    xx, yy = xr @ xr, yr @ yr
    if xx < 1e-12 or yy < 1e-12 or n-g-q.shape[1]-1 <= 0:
        return np.nan, np.nan, n, g
    beta = (xr @ yr)/xx
    error = yr-beta*xr
    scores = np.bincount(groups, weights=xr*error)
    correction = g/(g-1)*(n-1)/(n-g-q.shape[1]-1)
    variance = correction*(scores @ scores)/xx**2
    pvalue = 2*t.sf(abs(beta)/np.sqrt(max(variance, 1e-30)), g-1)
    return float((xr @ yr)/np.sqrt(xx*yy)), float(pvalue), n, g


def bootstrap_spearman(x, y, replicates=2000, seed=42):
    rng = np.random.default_rng(seed)
    index = rng.integers(0, len(x), size=(replicates, len(x)))
    rx, ry = normalized_ranks(np.asarray(x)[index], axis=1), normalized_ranks(np.asarray(y)[index], axis=1)
    valid = ((rx**2).sum(1) > 0) & ((ry**2).sum(1) > 0)
    values = (rx[valid]*ry[valid]).sum(1)
    if len(values) < 100:
        return np.nan, np.nan, int(valid.sum())
    lo, hi = np.percentile(values, [2.5, 97.5])
    return float(lo), float(hi), int(valid.sum())