import glob from typing import List, Mapping, Optional, Any, Tuple import collections import os import pickle from matplotlib.patches import Patch #from rpy2.robjects import conversion, default_converter import matplotlib.pyplot as plt import numpy as np import pandas as pd import shap import os # Attempt to set R_HOME manually (replace "/usr/lib/R" with your R home path if different) os.environ['R_HOME'] = '/usr/lib/R' # You might also need to add R to the PATH os.environ['PATH'] += os.pathsep + '/usr/bin' import rpy2.robjects as robjects from rpy2.robjects import pandas2ri from rpy2.robjects.conversion import localconverter import xgboost as xgb from matplotlib.patches import Patch """ Author: Intae Moon This script contains utility functions for predictions and interpretations. """ def get_xgboost_cancer_type_preds(xgb_model: xgb.sklearn.XGBClassifier, features_test_df: pd.DataFrame, cancer_types: List[str]) -> pd.DataFrame: """Returns cancer type predictions for test set using XGBoost model. Args: xgb_model: XGBoost model. features_test_df: Test set features. cancer_types: List of cancer types. Returns: pd.DataFrame containing cancer type predictions and prediction probabilities. """ ckp_test_pred_probs = xgb_model.predict_proba(features_test_df.values) ckp_test_preds = ckp_test_pred_probs.argmax(axis=1) max_posteriors = [pred_dist[max_idx] for pred_dist, max_idx in zip(ckp_test_pred_probs, ckp_test_preds)] ckp_test_preds_df = pd.DataFrame(ckp_test_pred_probs, index=features_test_df.index, columns=cancer_types) ckp_test_preds_df['max_posterior'] = max_posteriors ckp_test_preds_df['cancer_type'] = [cancer_types[max_idx] for max_idx in ckp_test_preds] return ckp_test_preds_df def get_xgboost_latest_cancer_type_preds(xgb_model: xgb.core.Booster, features_test_df: pd.DataFrame, cancer_types: List[str]) -> pd.DataFrame: """Returns cancer type predictions for test set using XGBoost model. Args: xgb_model: XGBoost model. features_test_df: Test set features. cancer_types: List of cancer types. Returns: pd.DataFrame containing cancer type predictions and prediction probabilities. """ dtest = xgb.DMatrix(features_test_df.values) ckp_test_pred_probs = xgb_model.predict(dtest, output_margin=True) ckp_test_pred_probs = np.exp(ckp_test_pred_probs) ckp_test_pred_probs /= ckp_test_pred_probs.sum(axis=1, keepdims=True) ckp_test_preds = ckp_test_pred_probs.argmax(axis=1) max_posteriors = [pred_dist[max_idx] for pred_dist, max_idx in zip(ckp_test_pred_probs, ckp_test_preds)] ckp_test_preds_df = pd.DataFrame(ckp_test_pred_probs, index=features_test_df.index, columns=cancer_types) ckp_test_preds_df['max_posterior'] = max_posteriors ckp_test_preds_df['cancer_type'] = [cancer_types[max_idx] for max_idx in ckp_test_preds] return ckp_test_preds_df def obtain_shap_values(model: xgb.sklearn.XGBClassifier, data: pd.DataFrame) -> np.ndarray: """Returns SHAP values for predictions based on data. Args: model: XGBoost model. data: Data to obtain SHAP values for. Returns: Numpy array containing SHAP values. """ # Get SHAP values using the model in byte array. mybooster = model.get_booster() model_bytearray = mybooster.save_raw()[4:] def in_bytearray(self=None): return model_bytearray mybooster.save_raw = in_bytearray shap_ex = shap.TreeExplainer(mybooster) return shap_ex.shap_values(data) def obtain_shap_values_with_latest_xgboost(model: xgb.core.Booster, data: pd.DataFrame) -> np.ndarray: """Returns SHAP values for predictions based on data. Args: model: XGBoost model. data: Data to obtain SHAP values for. Returns: Numpy array containing SHAP values. """ # Directly use the Booster with the TreeExplainer shap_ex = shap.TreeExplainer(model) return shap_ex.shap_values(data) def partition_feature_names_by_group(fature_names: List[str]): """Partitions feature names into groups. Args: feature_names: List of feature names. Returns: Dictionary mapping feature groups to feature names and vice versa. """ feature_group_to_features_dict = collections.defaultdict(list) feature_to_feature_group_dict = {} for feat in fature_names: if 'SBS' in feat: feature_group_to_features_dict['signature'].append(feat) feature_to_feature_group_dict[feat] = 'signature' elif feat in ['Age', 'Sex']: feature_group_to_features_dict['clinical'].append(feat) feature_to_feature_group_dict[feat] = 'clinical' elif 'CNA' in feat: feature_group_to_features_dict['cna'].append(feat) feature_to_feature_group_dict[feat] = 'cna' else: feature_group_to_features_dict['mutation'].append(feat) feature_to_feature_group_dict[feat] = 'mutation' return feature_group_to_features_dict, feature_to_feature_group_dict def get_color(feature_name: str, feature_group_dict: Mapping[str, List[str]]) -> str: """ Determines the color for a given feature based on its group. Args: feature_name: Name of the feature. feature_group_dict: Dictionary mapping feature groups to feature names. Returns: A string representing the color associated with the feature's group. """ color_mapping = {'mutation': 'red', 'cna': 'green', 'signature': 'blue', 'clinical': 'grey'} for group, features in feature_group_dict.items(): if feature_name in features: return color_mapping[group] return 'black' # Default color if not found def get_individual_pred_interpretation(shap_pred_sample_df: pd.DataFrame, feature_sample_df: pd.DataFrame, feature_group_to_features_dict: dict, feature_to_feature_group_dict: dict, sample_info: str = None, filename: str = None, filepath: str = '../others_prediction_explanation', cohort_age_stats_path: str = '../data/combined_cohort_age_stats.pkl', top_feature_num: int = 10, save_plot: bool = False): """ Dynamic version of the function for individual prediction interpretation for a given tumor sample. Args: shap_pred_sample_df: DataFrame containing SHAP values for a given tumor sample. feature_sample_df: DataFrame containing feature values for a given tumor sample. feature_group_to_features_dict: Dictionary mapping feature groups to feature names. sample_info: Sample information to be displayed. filename: Filename to save the figure. top_feature_num: Number of top features to display. """ # Initialize plot plt.rcParams.update({'font.size': 15, "font.family": "Arial"}) fig, ax = plt.subplots() # remove top and right lines for bar graph ax.spines['right'].set_visible(False) ax.spines['top'].set_visible(False) # Sorting and selecting top features based on SHAP values all_features = sum(feature_group_to_features_dict.values(), []) sorted_features = sorted(all_features, key=lambda x: abs(shap_pred_sample_df.loc[x]), reverse=True) top_features = sorted(sorted_features[:top_feature_num], key=lambda x: shap_pred_sample_df.loc[x], reverse=False) # Preparing data for the bar chart top_feats_df = pd.DataFrame({ 'feat_name': top_features, 'SHAP_val': shap_pred_sample_df.loc[top_features], 'feat_val': feature_sample_df.loc[top_features], 'color': [get_color(feat, feature_group_to_features_dict) for feat in top_features] }) # Creating the bar chart ax.barh(top_feats_df['feat_name'], top_feats_df['SHAP_val'], color=top_feats_df['color']) ax.set_xlabel('SHAP Values') ax.set_title(sample_info) ax.set_yticks([]) with open(cohort_age_stats_path, "rb") as fp: combined_cohort_age_stats = pickle.load(fp) # Dynamic positioning of feature names and values left_margin = ax.get_xlim()[0] * 1.1 -.1 # Calculate the left margin dynamically for i, (name, value) in enumerate(zip(top_feats_df['feat_name'], top_feats_df['feat_val'])): value_text = f'{int(value)}' if feature_to_feature_group_dict[name] == 'mutation' else f'{value:.2f}' # Feature name and value if name == 'Sex': value_text = 'Male' if value == 1.0 else 'Female' if name == 'Age': value_text = int(value * combined_cohort_age_stats['Std_mean'] + combined_cohort_age_stats['Age_mean']) if name[-3:] == 'CNA': value_text = f'{int(value)}' ax.text(left_margin, i, f'{name}: {value_text}', ha='right', va='center', fontsize=10) ax.text(left_margin, top_feature_num, f'feature: value', ha='right', va='center', fontsize=10) # Adding legend legend_elements = [Patch(facecolor=color, label=label) for label, color in zip(['Somatic Mut.', 'CNA events', 'Mutation Sig.', 'Age/Sex'], ['red', 'green', 'blue', 'grey'])] ax.legend(handles=legend_elements, title='Feature Groups') plt.tight_layout() if save_plot: plt.savefig(f'{filepath}/{filename}.svg') plt.savefig(f'{filepath}/{filename}.pdf') plt.show() return f'{filepath}/{filename}.svg' def get_top_n_pred_and_shap( preds_df: pd.DataFrame, shaps: np.asarray, features: List[str], cancer_types: List[str], n: int=3) -> Mapping[str, Mapping[str, Any]]: """ Get top n predictions for each sample in the given DataFrame. Args: preds_df: DataFrame containing predictions. shaps: Array of SHAP values. features: List of features. n: Number of top predictions to return. cancer_types: List of cancer types. Returns: Dictionary containing top n predictions and SHAP values for each sample. """ # Filter only relevant columns (cancer types) df_chosen = preds_df[cancer_types] # Initialize the dictionary for SHAP values sample_id_to_top_n_pred_and_shaps_df = {} for sample_id, row in df_chosen.iterrows(): # Getting the top n predictions sorted by their probabilities sorted_top_3 = row.nlargest(n).sort_values(ascending=False) # Get row index for sample_id row_idx = preds_df.index.get_loc(sample_id) # Store the top n predictions and their probabilities shap_dfs = [] for pred_cancer in sorted_top_3.index: pred_cancer_idx = cancer_types.index(pred_cancer) shap_pred_cancer_df = pd.DataFrame(shaps[pred_cancer_idx][row_idx], index=features, columns=[pred_cancer]).T shap_dfs.append(shap_pred_cancer_df) shap_total_df = pd.concat(shap_dfs) sample_id_to_top_n_pred_and_shaps_df[sample_id] = { 'top_n_predictions': sorted_top_3.to_dict(), 'shap_values_df': shap_total_df } return sample_id_to_top_n_pred_and_shaps_df def get_onconpc_prediction_explanations(query_ids: List[str], preds_df: pd.DataFrame, shaps: np.array, df_features_genie: pd.DataFrame, cancer_types_to_consider: List[str], filepath: str='../others_prediction_explanation', cohort_age_stats_path: str='../data/combined_cohort_age_stats.pkl', save_plot: bool=False, ) -> List[Mapping[str, Any]]: """ Get OncoNPC predictions and generate SHAP-based explanation plots for multiple query IDs. Args: query_ids: List of IDs of the tumor samples to query. preds_df: DataFrame containing predictions. shaps: Array of SHAP values. df_features_genie: DataFrame containing features. cancer_types_to_consider: List of cancer types considered in the prediction. filepath: Path to save the explanation plots. save_plot: Whether to save the explanation plots. Returns: List of dictionaries containing prediction details and explanation plots for each query ID. """ results_dict = {} for query_id in query_ids: # Get OncoNPC prediction pred_prob = preds_df.at[query_id, 'max_posterior'] pred_cancer = preds_df.at[query_id, 'cancer_type'] pred_cancer_idx = cancer_types_to_consider.index(pred_cancer) # Get SHAP-based explanation for the prediction feature_sample_df = df_features_genie.loc[query_id] shap_pred_cancer_df = pd.DataFrame(shaps[pred_cancer_idx], index=df_features_genie.index, columns=df_features_genie.columns) shap_pred_sample_df = shap_pred_cancer_df.loc[query_id] # Information and plot generation sample_info = f'SAMPLE_ID: {query_id}\nPrediction: {pred_cancer}\nPrediction probability: {pred_prob:.3f}' feature_group_to_features_dict, feature_to_feature_group_dict = partition_feature_names_by_group(df_features_genie.columns) full_filename = get_individual_pred_interpretation(shap_pred_sample_df, feature_sample_df, feature_group_to_features_dict, feature_to_feature_group_dict, sample_info=sample_info, filename=str(query_id), filepath=filepath, top_feature_num=10, cohort_age_stats_path=cohort_age_stats_path, save_plot=save_plot) # Store the results results_dict[query_id] = { 'pred_prob': pred_prob, 'pred_cancer': pred_cancer, 'explanation_plot': full_filename } return results_dict def get_onconpc_features_from_raw_data(df_patients_chosen: pd.DataFrame, df_samples_chosen: pd.DataFrame, df_mutations_chosen: pd.DataFrame, df_cna_chosen: pd.DataFrame, features_onconpc_path: str='data/features_onconpc.pkl', combined_cohort_age_stats_path: str='data/combined_cohort_age_stats.pkl', mut_sig_weights_filepath: str='data/mutation_signatures/sigProfiler*.csv' ) -> Tuple[pd.DataFrame, pd.DataFrame]: """ Process raw GENIE data for OncoNPC prediction inference. Args: df_patients_chosen: DataFrame containing chosen patient data. df_samples_chosen: DataFrame containing chosen sample data. df_mutations_chosen: DataFrame containing chosen mutation data. df_cna_chosen: DataFrame containing chosen copy number alteration data. features_onconpc_path: Path to OncoNPC features file. combined_cohort_age_stats_path: Path to combined cohort age statistics file. weights_filepath: Path to mutation signatures weights file. Returns: Processed DataFrame ready for OncoNPC prediction inference. """ # Re-shape df_cna_chosen df_cna_reshaped = df_cna_chosen.set_index(['Hugo_Symbol']).T.copy() # Merge patient and sample data df_patients_merged = pd.merge(df_patients_chosen, df_samples_chosen, how='right', on='PATIENT_ID') # Process mutation data df_trinuc_feats_genie = get_snv_in_trinuc_context(df_mutations_chosen, sample_id_col='Tumor_Sample_Barcode', chromosome_col='Chromosome', start_pos_col='Start_Position', ref_allele_col='Reference_Allele', alt_allele_col='Tumor_Seq_Allele2', config='genie') # Obtain mutation signatures df_mut_sigs_genie = obtain_mutation_signatures(df_trinuc_feats_genie, weights_filepath=mut_sig_weights_filepath) # Preprocess features and labels df_features_genie, df_labels_genie = pre_process_features_genie(df_mutations_chosen, df_cna_reshaped, df_mut_sigs_genie, df_patients_merged) # Zero-pad missing features with open(features_onconpc_path, "rb") as fp: features_onconpc = pickle.load(fp) df_features_genie_final = zero_pad_missing_features(df_features_genie, features_onconpc) # Standardize feature names df_features_genie_final.columns = standardize_feat_names(df_features_genie_final.columns) # Standardize Age feature with open(combined_cohort_age_stats_path, "rb") as fp: combined_cohort_age_stats = pickle.load(fp) df_features_genie_final['Age'] = (df_features_genie_final['Age'] - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean'] return df_features_genie_final, df_labels_genie def get_snv_in_trinuc_context(df_mutations: pd.DataFrame, sample_id_col: str, chromosome_col: str, start_pos_col: str, ref_allele_col: str, alt_allele_col: str, config: Optional[str]='genie') -> pd.DataFrame: """ Processes the given mutations DataFrame to get SNV in trinucleotide context. Args: df_mutations: DataFrame containing mutation data. sample_id_col: Column name for sample IDs. chromosome_col: Column name for chromosome. start_pos_col: Column name for start position. ref_allele_col: Column name for reference allele. alt_allele_col: Column name for alternative allele. Returns: DataFrame with SNV in trinucleotide context. """ # Specify the columns to be chosen columns_chosen = [sample_id_col, chromosome_col, start_pos_col, ref_allele_col, alt_allele_col] # Process the DataFrame df_mutations_chosen_trinuc = df_mutations[columns_chosen].copy() if config == 'genie': df_mutations_chosen_trinuc[chromosome_col] = 'chr' + df_mutations_chosen_trinuc[chromosome_col].astype(str) df_mutations_chosen_trinuc = df_mutations_chosen_trinuc[~df_mutations_chosen_trinuc[chromosome_col].str.contains('GL|chrMT')] # Change the data types for R processing df_mutations_chosen_trinuc[sample_id_col] = df_mutations_chosen_trinuc[sample_id_col].astype('str') # R function code r_code = ''' function(mutationData, sample_id, chr, pos, ref, alt) { library(deconstructSigs) sigs.input <- mut.to.sigs.input(mut.ref = mutationData, sample.id = sample_id, chr = chr, pos = pos, ref = ref, alt = alt) # Filter samples with low mutations sigs.input <- sigs.input[rowSums(sigs.input) >= 1,] return(sigs.input) } ''' # Convert the DataFrame for R processing with conversion.localconverter(default_converter): with localconverter(robjects.default_converter + pandas2ri.converter): r_df_mutations = robjects.conversion.py2rpy(df_mutations_chosen_trinuc) # Load and call the R function r_function = robjects.r(r_code) with conversion.localconverter(default_converter): with localconverter(robjects.default_converter + pandas2ri.converter): df_trinuc_feats = robjects.conversion.rpy2py(r_function(r_df_mutations, sample_id_col, chromosome_col, start_pos_col, ref_allele_col, alt_allele_col)) if config=='profile_dfci': df_trinuc_feats.index = df_trinuc_feats.index.astype('float') return df_trinuc_feats def obtain_mutation_signatures(df_trinuc_feats: pd.DataFrame, weights_filepath: str='../data/mutation_signatures/sigProfiler*.csv' ) -> pd.DataFrame: """Transforms tri-nucleotide features into mutation signature based features. Args: df_trinuc_feats: DataFrame containing tri-nucleotide features. Returns: mut_sig_based_df: DataFrame containing mutation signature based features. """ # Load Mutational Signatures COSMIC data. file_names = glob.glob(weights_filepath) # Check if the directory exists. if len(file_names) == 0: raise ValueError('No mutation signatures data.') file_names_key = [file_name.split('_')[-1][:-4] for file_name in file_names] sbs_conversion_df_dict = {} for i in range(len(file_names)): df = pd.read_csv(file_names[i]) df.dropna(inplace = True) # process base names s.t. they compatible with actual feature names new_name_list = [] for change, subtype in zip(df.Type.values, df.Subtype.values): new_name = subtype[0] + '[' + change + ']' + subtype[2] new_name_list.append(new_name) df['Type_new'] = new_name_list df.set_index('Type_new', inplace = True) # check if column contains human-based experiment results merged_colum_names = ''.join(df.columns) if 'GRCh38' in merged_colum_names: df['values_oi'] = df[file_names_key[i] + '_GRCh38'].values elif 'GRCh37' in merged_colum_names: df['values_oi'] = df[file_names_key[i] + '_GRCh37'].values else: raise ValueError('No human-based experimental results.') sum_vals = np.sum(df.values_oi.values) if abs(sum_vals - 1.0) > 0.1: raise ValueError('Values do not sum up to 1.') sbs_conversion_df_dict[file_names_key[i]] = df mut_sig_based_df = pd.DataFrame([], index = df_trinuc_feats.index) for key, mut_df in sbs_conversion_df_dict.items(): common_substitute_feats = list(set(mut_df.index) & set(df_trinuc_feats.columns)) if len(common_substitute_feats) != 96: raise ValueError('Number of substitution features does not sum up to 96.') # Get mutation signature values mut_sig_vals = np.matmul(df_trinuc_feats[common_substitute_feats].values, mut_df.loc[common_substitute_feats].values_oi.values) mut_sig_based_df[key] = mut_sig_vals return mut_sig_based_df def pre_process_features_genie(df_mutations: pd.DataFrame, df_cna: pd.DataFrame, df_mutation_signatures: pd.DataFrame, df_patients: pd.DataFrame, cancer_types: Optional[List[str]]=None, id_column: str = 'SAMPLE_ID', cancer_type_column: str='CANCER_TYPE') -> Tuple[pd.DataFrame, pd.DataFrame]: """Pre-process genetics data to create feature df for GENIE. Args: df_mutations: DataFrame containing mutation data. df_cna: DataFrame containing CNA data. df_mutation_signatures: DataFrame containing mutation signature data. df_patients: DataFrame containing patient data. cancer_types: List of cancer types. id_column: Column name for sample IDs. cancer_type_column: Column name for cancer types. Returns: df_features_merged_final: DataFrame containing merged features. df_labels_final: DataFrame containing labels. """ # Get mutation features. sample_ids = df_patients[id_column].values df_mutations_chosen = df_mutations.loc[df_mutations.Tumor_Sample_Barcode.isin(sample_ids)] mutation_gene_names = np.unique(df_mutations_chosen.Hugo_Symbol.values) df_mutation_feature = pd.DataFrame(0, columns=mutation_gene_names, index=sample_ids) for entry in df_mutations_chosen[['Tumor_Sample_Barcode', 'Hugo_Symbol']].values: sample_id = entry[0] gene = entry[1] if gene in df_mutation_feature.columns: # Count the number of mutation frequency in each gene. df_mutation_feature.at[sample_id, gene] = df_mutation_feature.at[sample_id, gene] + 1 # Add 'mut' at the end of each mutation feature df_mutation_feature.columns = [gene + '_mut' for gene in mutation_gene_names] # Get CNA features. df_cna_chosen = df_cna.loc[list(set(df_cna.index) & set(sample_ids))] df_cna_chosen.fillna(0, inplace = True) # Obtain age/sex features. df_patients_id_indexed = df_patients.set_index('SAMPLE_ID') df_sex_age_feature = df_patients_id_indexed[['SEX', 'AGE_AT_SEQ_REPORT']].copy() sex_list = [] age_nan_indices = [] age_list = [] for sample_id, (sex, age) in zip(df_sex_age_feature.index, df_sex_age_feature.values): if sex == 'Male': sex_list.append(1) elif sex == 'Female': sex_list.append(-1) else: # Not reported sex_list.append(0) if pd.isnull(age): age_nan_indices.append(sample_id) age_list.append(0) # append zero for now elif '>89' in str(age): age_list.append(int(age[1:])) elif '<18' in str(age): age_list.append(int(age[1:])) else: age_list.append(int(age)) df_sex_age_feature['SEX'] = sex_list df_sex_age_feature['AGE_AT_SEQ_REPORT'] = age_list df_sex_age_feature.columns = ['Sex', 'Age'] # Merge all feature dfs using common sample IDs. common_ids = list(set(df_mutation_feature.index) & set(df_cna_chosen.index) & set(df_mutation_signatures.index) & set(df_sex_age_feature.index)) dfs_list = [df_mutation_feature, df_cna_chosen, df_mutation_signatures, df_sex_age_feature] df_features_merged = merge_dfs(dfs_list, common_ids) # Exclude age NaN indices. df_features_merged_final = df_features_merged.loc[list(set(df_features_merged.index) - set(age_nan_indices))] df_labels = df_patients_id_indexed.loc[df_features_merged_final.index][cancer_type_column] if cancer_types is not None: df_labels_final = pd.DataFrame([cancer_types.index(val) for val in df_labels.values], columns = ['cancer_label'], index = df_labels.index) df_labels_final['cancer_type'] = df_labels.values return df_features_merged_final, df_labels_final else: return df_features_merged_final, df_labels def pre_process_features_dfci(df_mutations: pd.DataFrame, df_cna: pd.DataFrame, df_mutation_signatures: pd.DataFrame, df_patients: pd.DataFrame, cancer_types: List[str], cup_samples_ids: Optional[List[str]]=None, id_column: str = 'UNIQUE_SAMPLE_ID' ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: """Pre-process genetics data to create feature df for DFCI data. Args: df_mutations: DataFrame containing mutation data. df_cna: DataFrame containing CNA data. df_mutation_signatures: DataFrame containing mutation signature data. df_patients: DataFrame containing patient data. cancer_types: List of cancer types. cup_samples_ids: List of CUP sample IDs. id_column: Column name for sample IDs. Returns: df_features_merged_ckps: DataFrame containing merged features for CKP samples. df_labels_final: DataFrame containing labels for CKP samples. df_features_merged_cups: DataFrame containing merged features for CUP samples. """ # Get mutation features. sample_ids = df_patients[id_column].values df_mutations_chosen = df_mutations.loc[df_mutations[id_column].isin(sample_ids)] mutation_gene_names = np.unique(df_mutations_chosen.CANONICAL_GENE.values) df_mutation_feature = pd.DataFrame(0, columns=mutation_gene_names, index=sample_ids) for entry in df_mutations_chosen[['UNIQUE_SAMPLE_ID', 'CANONICAL_GENE']].values: sample_id = entry[0]; gene = entry[1]; if gene in df_mutation_feature.columns: # Count the number of mutation frequency in each gene. df_mutation_feature.at[sample_id, gene] = df_mutation_feature.at[sample_id, gene] + 1 # Add 'mut' at the end of each mutation feature df_mutation_feature.columns = [gene + '_mut' for gene in mutation_gene_names] # Get CNA features. df_cna_chosen = df_cna.loc[list(set(df_cna.index) & set(sample_ids))] df_cna_chosen.fillna(0, inplace = True) # Obtain age/sex features. df_patients_id_indexed = df_patients.set_index(id_column) df_sex_age_feature = df_patients_id_indexed[['Sex', 'Age']].copy() sex_list = [] for sample_id, (sex, age) in zip(df_sex_age_feature.index, df_sex_age_feature.values): if sex == 'MALE': sex_list.append(1) elif sex == 'FEMALE': sex_list.append(-1) else: # Not reported sex_list.append(0) df_sex_age_feature['Sex'] = sex_list df_sex_age_feature.columns = ['Sex', 'Age'] # Merge all features using the same sample IDs. common_ids = list(set(df_mutation_feature.index) & set(df_cna_chosen.index) & set(df_mutation_signatures.index) & set(df_sex_age_feature.index)) dfs_list = [df_mutation_feature, df_cna_chosen, df_mutation_signatures, df_sex_age_feature] df_features_merged = merge_dfs(dfs_list, common_ids) # Make gene names consistent across GENIE and PROFILE. profile_old_new_gene_mapping = {} profile_old_new_gene_mapping['C17ORF70'] = 'FAAP100' profile_old_new_gene_mapping['C17orf70_mut'] = 'FAAP100_mut' profile_old_new_gene_mapping['C19ORF40'] = 'FAAP24' profile_old_new_gene_mapping['C19orf40_mut'] = 'FAAP24_mut' profile_old_new_gene_mapping['LOC96610_mut'] = 'BMS1P20_mut' profile_old_new_gene_mapping['LOC729991-MEF2B_mut'] = 'MEF2BNB-MEF2B_mut' profile_old_new_gene_mapping['C1orf86_mut'] = 'FAAP20_mut' profile_old_new_gene_mapping['C1ORF86'] = 'FAAP20' profile_old_new_gene_mapping['GNB2L1_mut'] = 'RACK1_mut' profile_old_new_gene_mapping['GNB2L1'] = 'RACK1' df_features_merged.rename(columns=profile_old_new_gene_mapping, inplace=True) df_features_merged_ckps = (df_features_merged .loc[list(set(df_features_merged.index) - set(cup_samples_ids))]) df_features_merged_cups = (df_features_merged .loc[list(set(df_features_merged.index) & set(cup_samples_ids))]) df_labels = df_patients_id_indexed.loc[df_features_merged_ckps.index]['CANCER_TYPE'] df_labels_final = pd.DataFrame([cancer_types.index(val) for val in df_labels.values], columns=['cancer_label'], index=df_labels.index) df_labels_final['cancer_type'] = df_labels.values return df_features_merged_ckps, df_labels_final, df_features_merged_cups def zero_pad_missing_features(df: pd.DataFrame, feature_list: List[str]) -> pd.DataFrame: """ Zero pad missing features. Args: df: DataFrame containing features. feature_list: List of features to include. Returns: df: DataFrame containing zero-padded features. """ # Create a dictionary for new columns to add new_columns = {feature: [0] * len(df) for feature in feature_list if feature not in df.columns} # Concatenate new columns with the original DataFrame df = pd.concat([df, pd.DataFrame(new_columns, index=df.index)], axis=1) return df[feature_list] def merge_dfs(df_list: pd.DataFrame, ids_common: List[str]) -> pd.DataFrame: """Merges a list of dataframes using common sample IDs. Args: df_list: List of dataframes to merge. ids_common: Common sample IDs. Returns: df_merged_to_return: Merged dataframe. """ for idx, df in enumerate(df_list): if idx == 0: df_merged_to_return = df.loc[ids_common] else: df_merged_to_return = pd.merge(df_merged_to_return, df.loc[ids_common], how = 'left', left_index = True, right_index = True) return df_merged_to_return def standardize_feat_names(curr_feat_names: List[str]) -> List[str]: """ Standardize feature names. Args: curr_feat_names: list of feature names Returns: new_feat_names: list of standardized feature names """ new_feat_names = [] for feat in curr_feat_names: if '_mut' in feat: new_feat_names.append(feat.replace('_mut', '')) elif 'AGE' in feat or 'Age' in feat: new_feat_names.append('Age') elif 'GENDER' in feat or 'Sex' in feat: new_feat_names.append('Sex') elif 'SBS' in feat: new_feat_names.append(feat) else: new_feat_names.append(feat + ' CNA') return new_feat_names