"""Clinical/proteomics loading and causal feature construction.""" from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Sequence, Tuple import numpy as np import pandas as pd TARGETS = ["updrs_1", "updrs_2", "updrs_3", "updrs_4"] HORIZONS = [0, 6, 12, 24] def _uid(is_supplemental: int, patient_id: Any) -> str: return ("s" if int(is_supplemental) else "t") + "_" + str(patient_id) def load_clinical(root: str | Path, include_supplemental: bool = False) -> pd.DataFrame: root = Path(root) train = pd.read_csv(root / "train_clinical_data.csv") train["is_supplemental"] = 0 frames = [train] if include_supplemental: supp = pd.read_csv(root / "supplemental_clinical_data.csv") supp["is_supplemental"] = 1 frames.append(supp) df = pd.concat(frames, ignore_index=True) df["uid"] = [ _uid(s, p) for s, p in zip(df["is_supplemental"].values, df["patient_id"].values) ] df = df.sort_values(["uid", "visit_month"]).reset_index(drop=True) return df def add_cohorts(df: pd.DataFrame) -> pd.DataFrame: """Add rule-based cohort labels. control: first non-zero visit month is 12 (the red group from the 5th place solution) frequent: first non-zero visit < 12 and a 6-month visit exists annual: everything else """ df = df.copy() grouped = df.groupby("uid", sort=False) visit_lists = grouped["visit_month"].apply(lambda s: sorted(set(int(x) for x in s))) rows = [] for uid, visits in visit_lists.items(): first_nz = next((v for v in visits if v > 0), 999) has6 = 6 in visits if first_nz == 12: cohort = "control" elif first_nz == 3: cohort = "frequent" else: cohort = "annual" rows.append( { "uid": uid, "cohort": cohort, "first_visit": min(visits) if visits else 999, "first_nonzero_visit": first_nz, "last_visit": max(visits) if visits else 999, "n_visits": len(visits), "has6": int(has6), } ) meta = pd.DataFrame(rows) df = df.merge(meta, on="uid", how="left") return df def patient_table(df: pd.DataFrame) -> pd.DataFrame: rows = [] for uid, g in df.groupby("uid", sort=False): r = g.iloc[0] rows.append( { "uid": uid, "patient_id": r["patient_id"], "is_supplemental": int(r["is_supplemental"]), "cohort": r["cohort"], "n_visits": int(r["n_visits"]), "first_visit": int(r["first_visit"]), "first_nonzero_visit": int(r["first_nonzero_visit"]), "last_visit": int(r["last_visit"]), "has6": int(r["has6"]), } ) return pd.DataFrame(rows).sort_values(["is_supplemental", "cohort", "uid"]).reset_index(drop=True) def load_proteins(root: str | Path) -> pd.DataFrame: root = Path(root) df = pd.read_csv(root / "train_proteins.csv") pivot = df.pivot_table( index="visit_id", columns="UniProt", values="NPX", aggfunc="mean" ) pivot = pivot.reindex(sorted(pivot.columns), axis=1) return pivot def load_peptides(root: str | Path, mode: str = "protein_mean") -> pd.DataFrame: root = Path(root) df = pd.read_csv( root / "train_peptides.csv", usecols=["visit_id", "UniProt", "Peptide", "PeptideAbundance"], ) if mode == "protein_mean": pivot = df.groupby(["visit_id", "UniProt"], observed=True)["PeptideAbundance"].mean().unstack() pivot = pivot.reindex(sorted(pivot.columns), axis=1) return pivot if mode == "peptide": pivot = df.pivot_table( index="visit_id", columns="Peptide", values="PeptideAbundance", aggfunc="mean" ) pivot = pivot.reindex(sorted(pivot.columns), axis=1) return pivot raise ValueError(f"Unknown peptide mode: {mode}") def build_base_visits( clinical: pd.DataFrame, protein_visits: Sequence[str], base_visits: str = "protein" ) -> pd.DataFrame: if base_visits == "protein": out = clinical[clinical["visit_id"].isin(set(protein_visits))].copy() elif base_visits == "all": out = clinical.copy() else: raise ValueError(f"Unknown base_visits: {base_visits}") return out.reset_index(drop=True) def build_samples( clinical: pd.DataFrame, base: pd.DataFrame, targets: Sequence[str], horizons: Sequence[int], protein_visits: Sequence[str], ) -> pd.DataFrame: lookup: Dict[Tuple[str, int], pd.Series] = {} for row in clinical.itertuples(index=False): lookup[(row.uid, int(row.visit_month))] = row protein_set = set(protein_visits) records: List[Dict[str, Any]] = [] for row in base.itertuples(index=False): t = int(row.visit_month) rec: Dict[str, Any] = { "uid": row.uid, "patient_id": row.patient_id, "visit_id": row.visit_id, "visit_month": t, "cohort": row.cohort, "is_supplemental": int(row.is_supplemental), "has_protein": int(row.visit_id in protein_set), } for target in targets: for h in horizons: key = (row.uid, t + h) hit = lookup.get(key) if hit is None: val = np.nan else: val = getattr(hit, target) rec[f"y_{target}_{h}"] = float(val) if pd.notna(val) else np.nan rec[f"m_{target}_{h}"] = int(pd.notna(val)) records.append(rec) return pd.DataFrame(records) def build_process_features( samples: pd.DataFrame, clinical: pd.DataFrame, cfg: Dict[str, Any], protein_visits: Sequence[str] | None = None, ) -> Tuple[np.ndarray, List[str]]: months = [int(x) for x in cfg.get("months", [0, 6, 12, 18, 24, 36, 48, 60, 72, 84])] visit_source = str(cfg.get("visit_source", "clinical")) if visit_source == "sample": if protein_visits is None: raise ValueError("visit_source=sample requires protein_visits") hist = clinical[clinical["visit_id"].isin(set(protein_visits))] elif visit_source == "clinical": hist = clinical else: raise ValueError(f"Unknown process.visit_source: {visit_source}") include_control = bool(cfg.get("include_control", True)) include_gap = bool(cfg.get("include_gap", True)) include_prior = bool(cfg.get("include_prior_count", True)) include_month = bool(cfg.get("include_visit_month", True)) visit_lists: Dict[str, List[int]] = {} for uid, g in hist.groupby("uid", sort=False): visit_lists[uid] = sorted(set(int(x) for x in g["visit_month"].values)) names: List[str] = [f"v{m}" for m in months] if include_control: names.append("control") if include_gap: names.append("gap") if include_prior: names.append("prior_count") if include_month: names.append("visit_month") rows: List[List[float]] = [] for row in samples.itertuples(index=False): uid = row.uid t = int(row.visit_month) visits = visit_lists.get(uid, []) visit_set = set(visits) first_nz = next((v for v in visits if v > 0), 999) vals: List[float] = [] for m in months: if m < t: vals.append(1.0 if m in visit_set else 0.0) elif m == t: vals.append(1.0) else: vals.append(-1.0) if include_control: if t < 12: vals.append(-1.0) elif first_nz == 12: vals.append(1.0) else: vals.append(0.0) if include_gap: prev = [v for v in visits if v < t] vals.append(float(t - max(prev)) if prev else -1.0) if include_prior: vals.append(float(sum(1 for v in visits if v < t))) if include_month: vals.append(float(t)) rows.append(vals) return np.asarray(rows, dtype=np.float32), names def build_protein_features( samples: pd.DataFrame, protein_df: pd.DataFrame | None, cfg: Dict[str, Any], prefix: str = "prot", ) -> Tuple[np.ndarray, List[str]]: if protein_df is None: raise ValueError("protein feature requested but protein table is None") cols = list(protein_df.columns) index = protein_df.index values: List[np.ndarray] = [] log1p = bool(cfg.get("log1p", True)) for visit_id in samples["visit_id"].values: if visit_id in index: vec = protein_df.loc[visit_id].to_numpy(dtype=np.float64, copy=True) else: vec = np.full(len(cols), np.nan, dtype=np.float64) if log1p: with np.errstate(invalid="ignore"): vec = np.log1p(np.where(vec >= 0, vec, np.nan)) values.append(vec) X = np.vstack(values).astype(np.float32) if values else np.zeros((0, len(cols)), dtype=np.float32) names = [f"{prefix}_{c}" for c in cols] return X, names def build_feature_matrix( clinical: pd.DataFrame, samples: pd.DataFrame, protein_df: pd.DataFrame | None, peptide_df: pd.DataFrame | None, data_cfg: Dict[str, Any], protein_visits: Sequence[str] | None = None, ) -> Tuple[np.ndarray, List[str]]: parts: List[np.ndarray] = [] names: List[str] = [] if data_cfg.get("process", {}).get("enabled", True): X, n = build_process_features(samples, clinical, data_cfg["process"], protein_visits=protein_visits) parts.append(X) names.extend(n) if data_cfg.get("proteins", {}).get("enabled", False): X, n = build_protein_features(samples, protein_df, data_cfg["proteins"], prefix="prot") parts.append(X) names.extend(n) if data_cfg.get("peptides", {}).get("enabled", False): X, n = build_protein_features(samples, peptide_df, data_cfg["peptides"], prefix="pep") parts.append(X) names.extend(n) if not parts: raise ValueError("No feature blocks enabled") return np.concatenate(parts, axis=1).astype(np.float32), names @dataclass class DataBundle: clinical: pd.DataFrame patients: pd.DataFrame samples_all: pd.DataFrame samples_eval: pd.DataFrame X_all: np.ndarray X_eval: np.ndarray feature_names: List[str] protein_visits: List[str] target_cols: List[str] mask_cols: List[str] targets: List[str] = field(default_factory=lambda: list(TARGETS)) horizons: List[int] = field(default_factory=lambda: list(HORIZONS)) def build_all_data(cfg: Dict[str, Any]) -> DataBundle: data_cfg = cfg["data"] root = data_cfg["root"] clinical = load_clinical(root, include_supplemental=bool(data_cfg.get("include_supplemental", False))) clinical = add_cohorts(clinical) protein_df = load_proteins(root) peptide_df = None if data_cfg.get("peptides", {}).get("enabled", False): peptide_df = load_peptides(root, mode=data_cfg["peptides"].get("mode", "protein_mean")) protein_visits = list(protein_df.index) targets = list(data_cfg["targets"]) horizons = [int(h) for h in data_cfg["horizons"]] base_all = build_base_visits(clinical, protein_visits, data_cfg.get("base_visits", "protein")) base_eval = build_base_visits(clinical, protein_visits, "protein") samples_all = build_samples(clinical, base_all, targets, horizons, protein_visits) samples_eval = build_samples(clinical, base_eval, targets, horizons, protein_visits) X_all, names = build_feature_matrix(clinical, samples_all, protein_df, peptide_df, data_cfg, protein_visits=protein_visits) X_eval, names_eval = build_feature_matrix(clinical, samples_eval, protein_df, peptide_df, data_cfg, protein_visits=protein_visits) if names != names_eval: raise RuntimeError("Feature names differ between train and eval matrices") if data_cfg.get("has_protein_only", False) or data_cfg.get("proteins", {}).get("enabled", False) or data_cfg.get("peptides", {}).get("enabled", False): keep_all = samples_all["has_protein"].to_numpy() == 1 keep_eval = samples_eval["has_protein"].to_numpy() == 1 samples_all = samples_all.loc[keep_all].reset_index(drop=True) X_all = X_all[keep_all] samples_eval = samples_eval.loc[keep_eval].reset_index(drop=True) X_eval = X_eval[keep_eval] target_cols = [f"y_{t}_{h}" for t in targets for h in horizons] mask_cols = [f"m_{t}_{h}" for t in targets for h in horizons] return DataBundle( clinical=clinical, patients=patient_table(clinical), samples_all=samples_all, samples_eval=samples_eval, X_all=X_all, X_eval=X_eval, feature_names=names, protein_visits=protein_visits, target_cols=target_cols, mask_cols=mask_cols, targets=targets, horizons=horizons, )