fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 3)
04ae9bb verified
Raw History Blame Contribute Delete
13.2 kB
"""Clinical/proteomics loading and causal feature construction."""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Sequence, Tuple
import numpy as np
import pandas as pd
TARGETS = ["updrs_1", "updrs_2", "updrs_3", "updrs_4"]
HORIZONS = [0, 6, 12, 24]
def _uid(is_supplemental: int, patient_id: Any) -> str:
return ("s" if int(is_supplemental) else "t") + "_" + str(patient_id)
def load_clinical(root: str | Path, include_supplemental: bool = False) -> pd.DataFrame:
root = Path(root)
train = pd.read_csv(root / "train_clinical_data.csv")
train["is_supplemental"] = 0
frames = [train]
if include_supplemental:
supp = pd.read_csv(root / "supplemental_clinical_data.csv")
supp["is_supplemental"] = 1
frames.append(supp)
df = pd.concat(frames, ignore_index=True)
df["uid"] = [
_uid(s, p) for s, p in zip(df["is_supplemental"].values, df["patient_id"].values)
]
df = df.sort_values(["uid", "visit_month"]).reset_index(drop=True)
return df
def add_cohorts(df: pd.DataFrame) -> pd.DataFrame:
"""Add rule-based cohort labels.
control: first non-zero visit month is 12 (the red group from the 5th place solution)
frequent: first non-zero visit < 12 and a 6-month visit exists
annual: everything else
"""
df = df.copy()
grouped = df.groupby("uid", sort=False)
visit_lists = grouped["visit_month"].apply(lambda s: sorted(set(int(x) for x in s)))
rows = []
for uid, visits in visit_lists.items():
first_nz = next((v for v in visits if v > 0), 999)
has6 = 6 in visits
if first_nz == 12:
cohort = "control"
elif first_nz == 3:
cohort = "frequent"
else:
cohort = "annual"
rows.append(
{
"uid": uid,
"cohort": cohort,
"first_visit": min(visits) if visits else 999,
"first_nonzero_visit": first_nz,
"last_visit": max(visits) if visits else 999,
"n_visits": len(visits),
"has6": int(has6),
}
)
meta = pd.DataFrame(rows)
df = df.merge(meta, on="uid", how="left")
return df
def patient_table(df: pd.DataFrame) -> pd.DataFrame:
rows = []
for uid, g in df.groupby("uid", sort=False):
r = g.iloc[0]
rows.append(
{
"uid": uid,
"patient_id": r["patient_id"],
"is_supplemental": int(r["is_supplemental"]),
"cohort": r["cohort"],
"n_visits": int(r["n_visits"]),
"first_visit": int(r["first_visit"]),
"first_nonzero_visit": int(r["first_nonzero_visit"]),
"last_visit": int(r["last_visit"]),
"has6": int(r["has6"]),
}
)
return pd.DataFrame(rows).sort_values(["is_supplemental", "cohort", "uid"]).reset_index(drop=True)
def load_proteins(root: str | Path) -> pd.DataFrame:
root = Path(root)
df = pd.read_csv(root / "train_proteins.csv")
pivot = df.pivot_table(
index="visit_id", columns="UniProt", values="NPX", aggfunc="mean"
)
pivot = pivot.reindex(sorted(pivot.columns), axis=1)
return pivot
def load_peptides(root: str | Path, mode: str = "protein_mean") -> pd.DataFrame:
root = Path(root)
df = pd.read_csv(
root / "train_peptides.csv",
usecols=["visit_id", "UniProt", "Peptide", "PeptideAbundance"],
)
if mode == "protein_mean":
pivot = df.groupby(["visit_id", "UniProt"], observed=True)["PeptideAbundance"].mean().unstack()
pivot = pivot.reindex(sorted(pivot.columns), axis=1)
return pivot
if mode == "peptide":
pivot = df.pivot_table(
index="visit_id", columns="Peptide", values="PeptideAbundance", aggfunc="mean"
)
pivot = pivot.reindex(sorted(pivot.columns), axis=1)
return pivot
raise ValueError(f"Unknown peptide mode: {mode}")
def build_base_visits(
clinical: pd.DataFrame, protein_visits: Sequence[str], base_visits: str = "protein"
) -> pd.DataFrame:
if base_visits == "protein":
out = clinical[clinical["visit_id"].isin(set(protein_visits))].copy()
elif base_visits == "all":
out = clinical.copy()
else:
raise ValueError(f"Unknown base_visits: {base_visits}")
return out.reset_index(drop=True)
def build_samples(
clinical: pd.DataFrame,
base: pd.DataFrame,
targets: Sequence[str],
horizons: Sequence[int],
protein_visits: Sequence[str],
) -> pd.DataFrame:
lookup: Dict[Tuple[str, int], pd.Series] = {}
for row in clinical.itertuples(index=False):
lookup[(row.uid, int(row.visit_month))] = row
protein_set = set(protein_visits)
records: List[Dict[str, Any]] = []
for row in base.itertuples(index=False):
t = int(row.visit_month)
rec: Dict[str, Any] = {
"uid": row.uid,
"patient_id": row.patient_id,
"visit_id": row.visit_id,
"visit_month": t,
"cohort": row.cohort,
"is_supplemental": int(row.is_supplemental),
"has_protein": int(row.visit_id in protein_set),
}
for target in targets:
for h in horizons:
key = (row.uid, t + h)
hit = lookup.get(key)
if hit is None:
val = np.nan
else:
val = getattr(hit, target)
rec[f"y_{target}_{h}"] = float(val) if pd.notna(val) else np.nan
rec[f"m_{target}_{h}"] = int(pd.notna(val))
records.append(rec)
return pd.DataFrame(records)
def build_process_features(
samples: pd.DataFrame,
clinical: pd.DataFrame,
cfg: Dict[str, Any],
protein_visits: Sequence[str] | None = None,
) -> Tuple[np.ndarray, List[str]]:
months = [int(x) for x in cfg.get("months", [0, 6, 12, 18, 24, 36, 48, 60, 72, 84])]
visit_source = str(cfg.get("visit_source", "clinical"))
if visit_source == "sample":
if protein_visits is None:
raise ValueError("visit_source=sample requires protein_visits")
hist = clinical[clinical["visit_id"].isin(set(protein_visits))]
elif visit_source == "clinical":
hist = clinical
else:
raise ValueError(f"Unknown process.visit_source: {visit_source}")
include_control = bool(cfg.get("include_control", True))
include_gap = bool(cfg.get("include_gap", True))
include_prior = bool(cfg.get("include_prior_count", True))
include_month = bool(cfg.get("include_visit_month", True))
visit_lists: Dict[str, List[int]] = {}
for uid, g in hist.groupby("uid", sort=False):
visit_lists[uid] = sorted(set(int(x) for x in g["visit_month"].values))
names: List[str] = [f"v{m}" for m in months]
if include_control:
names.append("control")
if include_gap:
names.append("gap")
if include_prior:
names.append("prior_count")
if include_month:
names.append("visit_month")
rows: List[List[float]] = []
for row in samples.itertuples(index=False):
uid = row.uid
t = int(row.visit_month)
visits = visit_lists.get(uid, [])
visit_set = set(visits)
first_nz = next((v for v in visits if v > 0), 999)
vals: List[float] = []
for m in months:
if m < t:
vals.append(1.0 if m in visit_set else 0.0)
elif m == t:
vals.append(1.0)
else:
vals.append(-1.0)
if include_control:
if t < 12:
vals.append(-1.0)
elif first_nz == 12:
vals.append(1.0)
else:
vals.append(0.0)
if include_gap:
prev = [v for v in visits if v < t]
vals.append(float(t - max(prev)) if prev else -1.0)
if include_prior:
vals.append(float(sum(1 for v in visits if v < t)))
if include_month:
vals.append(float(t))
rows.append(vals)
return np.asarray(rows, dtype=np.float32), names
def build_protein_features(
samples: pd.DataFrame,
protein_df: pd.DataFrame | None,
cfg: Dict[str, Any],
prefix: str = "prot",
) -> Tuple[np.ndarray, List[str]]:
if protein_df is None:
raise ValueError("protein feature requested but protein table is None")
cols = list(protein_df.columns)
index = protein_df.index
values: List[np.ndarray] = []
log1p = bool(cfg.get("log1p", True))
for visit_id in samples["visit_id"].values:
if visit_id in index:
vec = protein_df.loc[visit_id].to_numpy(dtype=np.float64, copy=True)
else:
vec = np.full(len(cols), np.nan, dtype=np.float64)
if log1p:
with np.errstate(invalid="ignore"):
vec = np.log1p(np.where(vec >= 0, vec, np.nan))
values.append(vec)
X = np.vstack(values).astype(np.float32) if values else np.zeros((0, len(cols)), dtype=np.float32)
names = [f"{prefix}_{c}" for c in cols]
return X, names
def build_feature_matrix(
clinical: pd.DataFrame,
samples: pd.DataFrame,
protein_df: pd.DataFrame | None,
peptide_df: pd.DataFrame | None,
data_cfg: Dict[str, Any],
protein_visits: Sequence[str] | None = None,
) -> Tuple[np.ndarray, List[str]]:
parts: List[np.ndarray] = []
names: List[str] = []
if data_cfg.get("process", {}).get("enabled", True):
X, n = build_process_features(samples, clinical, data_cfg["process"], protein_visits=protein_visits)
parts.append(X)
names.extend(n)
if data_cfg.get("proteins", {}).get("enabled", False):
X, n = build_protein_features(samples, protein_df, data_cfg["proteins"], prefix="prot")
parts.append(X)
names.extend(n)
if data_cfg.get("peptides", {}).get("enabled", False):
X, n = build_protein_features(samples, peptide_df, data_cfg["peptides"], prefix="pep")
parts.append(X)
names.extend(n)
if not parts:
raise ValueError("No feature blocks enabled")
return np.concatenate(parts, axis=1).astype(np.float32), names
@dataclass
class DataBundle:
clinical: pd.DataFrame
patients: pd.DataFrame
samples_all: pd.DataFrame
samples_eval: pd.DataFrame
X_all: np.ndarray
X_eval: np.ndarray
feature_names: List[str]
protein_visits: List[str]
target_cols: List[str]
mask_cols: List[str]
targets: List[str] = field(default_factory=lambda: list(TARGETS))
horizons: List[int] = field(default_factory=lambda: list(HORIZONS))
def build_all_data(cfg: Dict[str, Any]) -> DataBundle:
data_cfg = cfg["data"]
root = data_cfg["root"]
clinical = load_clinical(root, include_supplemental=bool(data_cfg.get("include_supplemental", False)))
clinical = add_cohorts(clinical)
protein_df = load_proteins(root)
peptide_df = None
if data_cfg.get("peptides", {}).get("enabled", False):
peptide_df = load_peptides(root, mode=data_cfg["peptides"].get("mode", "protein_mean"))
protein_visits = list(protein_df.index)
targets = list(data_cfg["targets"])
horizons = [int(h) for h in data_cfg["horizons"]]
base_all = build_base_visits(clinical, protein_visits, data_cfg.get("base_visits", "protein"))
base_eval = build_base_visits(clinical, protein_visits, "protein")
samples_all = build_samples(clinical, base_all, targets, horizons, protein_visits)
samples_eval = build_samples(clinical, base_eval, targets, horizons, protein_visits)
X_all, names = build_feature_matrix(clinical, samples_all, protein_df, peptide_df, data_cfg, protein_visits=protein_visits)
X_eval, names_eval = build_feature_matrix(clinical, samples_eval, protein_df, peptide_df, data_cfg, protein_visits=protein_visits)
if names != names_eval:
raise RuntimeError("Feature names differ between train and eval matrices")
if data_cfg.get("has_protein_only", False) or data_cfg.get("proteins", {}).get("enabled", False) or data_cfg.get("peptides", {}).get("enabled", False):
keep_all = samples_all["has_protein"].to_numpy() == 1
keep_eval = samples_eval["has_protein"].to_numpy() == 1
samples_all = samples_all.loc[keep_all].reset_index(drop=True)
X_all = X_all[keep_all]
samples_eval = samples_eval.loc[keep_eval].reset_index(drop=True)
X_eval = X_eval[keep_eval]
target_cols = [f"y_{t}_{h}" for t in targets for h in horizons]
mask_cols = [f"m_{t}_{h}" for t in targets for h in horizons]
return DataBundle(
clinical=clinical,
patients=patient_table(clinical),
samples_all=samples_all,
samples_eval=samples_eval,
X_all=X_all,
X_eval=X_eval,
feature_names=names,
protein_visits=protein_visits,
target_cols=target_cols,
mask_cols=mask_cols,
targets=targets,
horizons=horizons,
)