Download code/ampd/data/clinical.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 13.2 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/data/clinical.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/data/clinical.py
-
curl -L -o clinical.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/data/clinical.py
13.2 kB
| """Clinical/proteomics loading and causal feature construction.""" | |
| from __future__ import annotations | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Sequence, Tuple | |
| import numpy as np | |
| import pandas as pd | |
| TARGETS = ["updrs_1", "updrs_2", "updrs_3", "updrs_4"] | |
| HORIZONS = [0, 6, 12, 24] | |
| def _uid(is_supplemental: int, patient_id: Any) -> str: | |
| return ("s" if int(is_supplemental) else "t") + "_" + str(patient_id) | |
| def load_clinical(root: str | Path, include_supplemental: bool = False) -> pd.DataFrame: | |
| root = Path(root) | |
| train = pd.read_csv(root / "train_clinical_data.csv") | |
| train["is_supplemental"] = 0 | |
| frames = [train] | |
| if include_supplemental: | |
| supp = pd.read_csv(root / "supplemental_clinical_data.csv") | |
| supp["is_supplemental"] = 1 | |
| frames.append(supp) | |
| df = pd.concat(frames, ignore_index=True) | |
| df["uid"] = [ | |
| _uid(s, p) for s, p in zip(df["is_supplemental"].values, df["patient_id"].values) | |
| ] | |
| df = df.sort_values(["uid", "visit_month"]).reset_index(drop=True) | |
| return df | |
| def add_cohorts(df: pd.DataFrame) -> pd.DataFrame: | |
| """Add rule-based cohort labels. | |
| control: first non-zero visit month is 12 (the red group from the 5th place solution) | |
| frequent: first non-zero visit < 12 and a 6-month visit exists | |
| annual: everything else | |
| """ | |
| df = df.copy() | |
| grouped = df.groupby("uid", sort=False) | |
| visit_lists = grouped["visit_month"].apply(lambda s: sorted(set(int(x) for x in s))) | |
| rows = [] | |
| for uid, visits in visit_lists.items(): | |
| first_nz = next((v for v in visits if v > 0), 999) | |
| has6 = 6 in visits | |
| if first_nz == 12: | |
| cohort = "control" | |
| elif first_nz == 3: | |
| cohort = "frequent" | |
| else: | |
| cohort = "annual" | |
| rows.append( | |
| { | |
| "uid": uid, | |
| "cohort": cohort, | |
| "first_visit": min(visits) if visits else 999, | |
| "first_nonzero_visit": first_nz, | |
| "last_visit": max(visits) if visits else 999, | |
| "n_visits": len(visits), | |
| "has6": int(has6), | |
| } | |
| ) | |
| meta = pd.DataFrame(rows) | |
| df = df.merge(meta, on="uid", how="left") | |
| return df | |
| def patient_table(df: pd.DataFrame) -> pd.DataFrame: | |
| rows = [] | |
| for uid, g in df.groupby("uid", sort=False): | |
| r = g.iloc[0] | |
| rows.append( | |
| { | |
| "uid": uid, | |
| "patient_id": r["patient_id"], | |
| "is_supplemental": int(r["is_supplemental"]), | |
| "cohort": r["cohort"], | |
| "n_visits": int(r["n_visits"]), | |
| "first_visit": int(r["first_visit"]), | |
| "first_nonzero_visit": int(r["first_nonzero_visit"]), | |
| "last_visit": int(r["last_visit"]), | |
| "has6": int(r["has6"]), | |
| } | |
| ) | |
| return pd.DataFrame(rows).sort_values(["is_supplemental", "cohort", "uid"]).reset_index(drop=True) | |
| def load_proteins(root: str | Path) -> pd.DataFrame: | |
| root = Path(root) | |
| df = pd.read_csv(root / "train_proteins.csv") | |
| pivot = df.pivot_table( | |
| index="visit_id", columns="UniProt", values="NPX", aggfunc="mean" | |
| ) | |
| pivot = pivot.reindex(sorted(pivot.columns), axis=1) | |
| return pivot | |
| def load_peptides(root: str | Path, mode: str = "protein_mean") -> pd.DataFrame: | |
| root = Path(root) | |
| df = pd.read_csv( | |
| root / "train_peptides.csv", | |
| usecols=["visit_id", "UniProt", "Peptide", "PeptideAbundance"], | |
| ) | |
| if mode == "protein_mean": | |
| pivot = df.groupby(["visit_id", "UniProt"], observed=True)["PeptideAbundance"].mean().unstack() | |
| pivot = pivot.reindex(sorted(pivot.columns), axis=1) | |
| return pivot | |
| if mode == "peptide": | |
| pivot = df.pivot_table( | |
| index="visit_id", columns="Peptide", values="PeptideAbundance", aggfunc="mean" | |
| ) | |
| pivot = pivot.reindex(sorted(pivot.columns), axis=1) | |
| return pivot | |
| raise ValueError(f"Unknown peptide mode: {mode}") | |
| def build_base_visits( | |
| clinical: pd.DataFrame, protein_visits: Sequence[str], base_visits: str = "protein" | |
| ) -> pd.DataFrame: | |
| if base_visits == "protein": | |
| out = clinical[clinical["visit_id"].isin(set(protein_visits))].copy() | |
| elif base_visits == "all": | |
| out = clinical.copy() | |
| else: | |
| raise ValueError(f"Unknown base_visits: {base_visits}") | |
| return out.reset_index(drop=True) | |
| def build_samples( | |
| clinical: pd.DataFrame, | |
| base: pd.DataFrame, | |
| targets: Sequence[str], | |
| horizons: Sequence[int], | |
| protein_visits: Sequence[str], | |
| ) -> pd.DataFrame: | |
| lookup: Dict[Tuple[str, int], pd.Series] = {} | |
| for row in clinical.itertuples(index=False): | |
| lookup[(row.uid, int(row.visit_month))] = row | |
| protein_set = set(protein_visits) | |
| records: List[Dict[str, Any]] = [] | |
| for row in base.itertuples(index=False): | |
| t = int(row.visit_month) | |
| rec: Dict[str, Any] = { | |
| "uid": row.uid, | |
| "patient_id": row.patient_id, | |
| "visit_id": row.visit_id, | |
| "visit_month": t, | |
| "cohort": row.cohort, | |
| "is_supplemental": int(row.is_supplemental), | |
| "has_protein": int(row.visit_id in protein_set), | |
| } | |
| for target in targets: | |
| for h in horizons: | |
| key = (row.uid, t + h) | |
| hit = lookup.get(key) | |
| if hit is None: | |
| val = np.nan | |
| else: | |
| val = getattr(hit, target) | |
| rec[f"y_{target}_{h}"] = float(val) if pd.notna(val) else np.nan | |
| rec[f"m_{target}_{h}"] = int(pd.notna(val)) | |
| records.append(rec) | |
| return pd.DataFrame(records) | |
| def build_process_features( | |
| samples: pd.DataFrame, | |
| clinical: pd.DataFrame, | |
| cfg: Dict[str, Any], | |
| protein_visits: Sequence[str] | None = None, | |
| ) -> Tuple[np.ndarray, List[str]]: | |
| months = [int(x) for x in cfg.get("months", [0, 6, 12, 18, 24, 36, 48, 60, 72, 84])] | |
| visit_source = str(cfg.get("visit_source", "clinical")) | |
| if visit_source == "sample": | |
| if protein_visits is None: | |
| raise ValueError("visit_source=sample requires protein_visits") | |
| hist = clinical[clinical["visit_id"].isin(set(protein_visits))] | |
| elif visit_source == "clinical": | |
| hist = clinical | |
| else: | |
| raise ValueError(f"Unknown process.visit_source: {visit_source}") | |
| include_control = bool(cfg.get("include_control", True)) | |
| include_gap = bool(cfg.get("include_gap", True)) | |
| include_prior = bool(cfg.get("include_prior_count", True)) | |
| include_month = bool(cfg.get("include_visit_month", True)) | |
| visit_lists: Dict[str, List[int]] = {} | |
| for uid, g in hist.groupby("uid", sort=False): | |
| visit_lists[uid] = sorted(set(int(x) for x in g["visit_month"].values)) | |
| names: List[str] = [f"v{m}" for m in months] | |
| if include_control: | |
| names.append("control") | |
| if include_gap: | |
| names.append("gap") | |
| if include_prior: | |
| names.append("prior_count") | |
| if include_month: | |
| names.append("visit_month") | |
| rows: List[List[float]] = [] | |
| for row in samples.itertuples(index=False): | |
| uid = row.uid | |
| t = int(row.visit_month) | |
| visits = visit_lists.get(uid, []) | |
| visit_set = set(visits) | |
| first_nz = next((v for v in visits if v > 0), 999) | |
| vals: List[float] = [] | |
| for m in months: | |
| if m < t: | |
| vals.append(1.0 if m in visit_set else 0.0) | |
| elif m == t: | |
| vals.append(1.0) | |
| else: | |
| vals.append(-1.0) | |
| if include_control: | |
| if t < 12: | |
| vals.append(-1.0) | |
| elif first_nz == 12: | |
| vals.append(1.0) | |
| else: | |
| vals.append(0.0) | |
| if include_gap: | |
| prev = [v for v in visits if v < t] | |
| vals.append(float(t - max(prev)) if prev else -1.0) | |
| if include_prior: | |
| vals.append(float(sum(1 for v in visits if v < t))) | |
| if include_month: | |
| vals.append(float(t)) | |
| rows.append(vals) | |
| return np.asarray(rows, dtype=np.float32), names | |
| def build_protein_features( | |
| samples: pd.DataFrame, | |
| protein_df: pd.DataFrame | None, | |
| cfg: Dict[str, Any], | |
| prefix: str = "prot", | |
| ) -> Tuple[np.ndarray, List[str]]: | |
| if protein_df is None: | |
| raise ValueError("protein feature requested but protein table is None") | |
| cols = list(protein_df.columns) | |
| index = protein_df.index | |
| values: List[np.ndarray] = [] | |
| log1p = bool(cfg.get("log1p", True)) | |
| for visit_id in samples["visit_id"].values: | |
| if visit_id in index: | |
| vec = protein_df.loc[visit_id].to_numpy(dtype=np.float64, copy=True) | |
| else: | |
| vec = np.full(len(cols), np.nan, dtype=np.float64) | |
| if log1p: | |
| with np.errstate(invalid="ignore"): | |
| vec = np.log1p(np.where(vec >= 0, vec, np.nan)) | |
| values.append(vec) | |
| X = np.vstack(values).astype(np.float32) if values else np.zeros((0, len(cols)), dtype=np.float32) | |
| names = [f"{prefix}_{c}" for c in cols] | |
| return X, names | |
| def build_feature_matrix( | |
| clinical: pd.DataFrame, | |
| samples: pd.DataFrame, | |
| protein_df: pd.DataFrame | None, | |
| peptide_df: pd.DataFrame | None, | |
| data_cfg: Dict[str, Any], | |
| protein_visits: Sequence[str] | None = None, | |
| ) -> Tuple[np.ndarray, List[str]]: | |
| parts: List[np.ndarray] = [] | |
| names: List[str] = [] | |
| if data_cfg.get("process", {}).get("enabled", True): | |
| X, n = build_process_features(samples, clinical, data_cfg["process"], protein_visits=protein_visits) | |
| parts.append(X) | |
| names.extend(n) | |
| if data_cfg.get("proteins", {}).get("enabled", False): | |
| X, n = build_protein_features(samples, protein_df, data_cfg["proteins"], prefix="prot") | |
| parts.append(X) | |
| names.extend(n) | |
| if data_cfg.get("peptides", {}).get("enabled", False): | |
| X, n = build_protein_features(samples, peptide_df, data_cfg["peptides"], prefix="pep") | |
| parts.append(X) | |
| names.extend(n) | |
| if not parts: | |
| raise ValueError("No feature blocks enabled") | |
| return np.concatenate(parts, axis=1).astype(np.float32), names | |
| class DataBundle: | |
| clinical: pd.DataFrame | |
| patients: pd.DataFrame | |
| samples_all: pd.DataFrame | |
| samples_eval: pd.DataFrame | |
| X_all: np.ndarray | |
| X_eval: np.ndarray | |
| feature_names: List[str] | |
| protein_visits: List[str] | |
| target_cols: List[str] | |
| mask_cols: List[str] | |
| targets: List[str] = field(default_factory=lambda: list(TARGETS)) | |
| horizons: List[int] = field(default_factory=lambda: list(HORIZONS)) | |
| def build_all_data(cfg: Dict[str, Any]) -> DataBundle: | |
| data_cfg = cfg["data"] | |
| root = data_cfg["root"] | |
| clinical = load_clinical(root, include_supplemental=bool(data_cfg.get("include_supplemental", False))) | |
| clinical = add_cohorts(clinical) | |
| protein_df = load_proteins(root) | |
| peptide_df = None | |
| if data_cfg.get("peptides", {}).get("enabled", False): | |
| peptide_df = load_peptides(root, mode=data_cfg["peptides"].get("mode", "protein_mean")) | |
| protein_visits = list(protein_df.index) | |
| targets = list(data_cfg["targets"]) | |
| horizons = [int(h) for h in data_cfg["horizons"]] | |
| base_all = build_base_visits(clinical, protein_visits, data_cfg.get("base_visits", "protein")) | |
| base_eval = build_base_visits(clinical, protein_visits, "protein") | |
| samples_all = build_samples(clinical, base_all, targets, horizons, protein_visits) | |
| samples_eval = build_samples(clinical, base_eval, targets, horizons, protein_visits) | |
| X_all, names = build_feature_matrix(clinical, samples_all, protein_df, peptide_df, data_cfg, protein_visits=protein_visits) | |
| X_eval, names_eval = build_feature_matrix(clinical, samples_eval, protein_df, peptide_df, data_cfg, protein_visits=protein_visits) | |
| if names != names_eval: | |
| raise RuntimeError("Feature names differ between train and eval matrices") | |
| if data_cfg.get("has_protein_only", False) or data_cfg.get("proteins", {}).get("enabled", False) or data_cfg.get("peptides", {}).get("enabled", False): | |
| keep_all = samples_all["has_protein"].to_numpy() == 1 | |
| keep_eval = samples_eval["has_protein"].to_numpy() == 1 | |
| samples_all = samples_all.loc[keep_all].reset_index(drop=True) | |
| X_all = X_all[keep_all] | |
| samples_eval = samples_eval.loc[keep_eval].reset_index(drop=True) | |
| X_eval = X_eval[keep_eval] | |
| target_cols = [f"y_{t}_{h}" for t in targets for h in horizons] | |
| mask_cols = [f"m_{t}_{h}" for t in targets for h in horizons] | |
| return DataBundle( | |
| clinical=clinical, | |
| patients=patient_table(clinical), | |
| samples_all=samples_all, | |
| samples_eval=samples_eval, | |
| X_all=X_all, | |
| X_eval=X_eval, | |
| feature_names=names, | |
| protein_visits=protein_visits, | |
| target_cols=target_cols, | |
| mask_cols=mask_cols, | |
| targets=targets, | |
| horizons=horizons, | |
| ) | |