Spaces:
Runtime error
Runtime error
Download codes/utils.py from jenniferzhou33/onconpc-visualize: direct link, hf CLI and curl.
- Browser
- Download file 30.3 kB
-
https://huggingface.co/spaces/jenniferzhou33/onconpc-visualize/resolve/main/codes/utils.py
- Command line
-
hf download hf://spaces/jenniferzhou33/onconpc-visualize/codes/utils.py
-
curl -L -o utils.py https://huggingface.co/spaces/jenniferzhou33/onconpc-visualize/resolve/main/codes/utils.py
30.3 kB
| import glob | |
| from typing import List, Mapping, Optional, Any, Tuple | |
| import collections | |
| import os | |
| import pickle | |
| from matplotlib.patches import Patch | |
| #from rpy2.robjects import conversion, default_converter | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| import pandas as pd | |
| import shap | |
| import os | |
| # Attempt to set R_HOME manually (replace "/usr/lib/R" with your R home path if different) | |
| os.environ['R_HOME'] = '/usr/lib/R' | |
| # You might also need to add R to the PATH | |
| os.environ['PATH'] += os.pathsep + '/usr/bin' | |
| import rpy2.robjects as robjects | |
| from rpy2.robjects import pandas2ri | |
| from rpy2.robjects.conversion import localconverter | |
| import xgboost as xgb | |
| from matplotlib.patches import Patch | |
| """ | |
| Author: Intae Moon | |
| This script contains utility functions for predictions and interpretations. | |
| """ | |
| def get_xgboost_cancer_type_preds(xgb_model: xgb.sklearn.XGBClassifier, | |
| features_test_df: pd.DataFrame, | |
| cancer_types: List[str]) -> pd.DataFrame: | |
| """Returns cancer type predictions for test set using XGBoost model. | |
| Args: | |
| xgb_model: XGBoost model. | |
| features_test_df: Test set features. | |
| cancer_types: List of cancer types. | |
| Returns: | |
| pd.DataFrame containing cancer type predictions and prediction probabilities. | |
| """ | |
| ckp_test_pred_probs = xgb_model.predict_proba(features_test_df.values) | |
| ckp_test_preds = ckp_test_pred_probs.argmax(axis=1) | |
| max_posteriors = [pred_dist[max_idx] for pred_dist, max_idx in | |
| zip(ckp_test_pred_probs, ckp_test_preds)] | |
| ckp_test_preds_df = pd.DataFrame(ckp_test_pred_probs, | |
| index=features_test_df.index, | |
| columns=cancer_types) | |
| ckp_test_preds_df['max_posterior'] = max_posteriors | |
| ckp_test_preds_df['cancer_type'] = [cancer_types[max_idx] for max_idx in ckp_test_preds] | |
| return ckp_test_preds_df | |
| def get_xgboost_latest_cancer_type_preds(xgb_model: xgb.core.Booster, | |
| features_test_df: pd.DataFrame, | |
| cancer_types: List[str]) -> pd.DataFrame: | |
| """Returns cancer type predictions for test set using XGBoost model. | |
| Args: | |
| xgb_model: XGBoost model. | |
| features_test_df: Test set features. | |
| cancer_types: List of cancer types. | |
| Returns: | |
| pd.DataFrame containing cancer type predictions and prediction probabilities. | |
| """ | |
| dtest = xgb.DMatrix(features_test_df.values) | |
| ckp_test_pred_probs = xgb_model.predict(dtest, output_margin=True) | |
| ckp_test_pred_probs = np.exp(ckp_test_pred_probs) | |
| ckp_test_pred_probs /= ckp_test_pred_probs.sum(axis=1, keepdims=True) | |
| ckp_test_preds = ckp_test_pred_probs.argmax(axis=1) | |
| max_posteriors = [pred_dist[max_idx] for pred_dist, max_idx in | |
| zip(ckp_test_pred_probs, ckp_test_preds)] | |
| ckp_test_preds_df = pd.DataFrame(ckp_test_pred_probs, | |
| index=features_test_df.index, | |
| columns=cancer_types) | |
| ckp_test_preds_df['max_posterior'] = max_posteriors | |
| ckp_test_preds_df['cancer_type'] = [cancer_types[max_idx] for max_idx in ckp_test_preds] | |
| return ckp_test_preds_df | |
| def obtain_shap_values(model: xgb.sklearn.XGBClassifier, | |
| data: pd.DataFrame) -> np.ndarray: | |
| """Returns SHAP values for predictions based on data. | |
| Args: | |
| model: XGBoost model. | |
| data: Data to obtain SHAP values for. | |
| Returns: | |
| Numpy array containing SHAP values. | |
| """ | |
| # Get SHAP values using the model in byte array. | |
| mybooster = model.get_booster() | |
| model_bytearray = mybooster.save_raw()[4:] | |
| def in_bytearray(self=None): | |
| return model_bytearray | |
| mybooster.save_raw = in_bytearray | |
| shap_ex = shap.TreeExplainer(mybooster) | |
| return shap_ex.shap_values(data) | |
| def obtain_shap_values_with_latest_xgboost(model: xgb.core.Booster, | |
| data: pd.DataFrame) -> np.ndarray: | |
| """Returns SHAP values for predictions based on data. | |
| Args: | |
| model: XGBoost model. | |
| data: Data to obtain SHAP values for. | |
| Returns: | |
| Numpy array containing SHAP values. | |
| """ | |
| # Directly use the Booster with the TreeExplainer | |
| shap_ex = shap.TreeExplainer(model) | |
| return shap_ex.shap_values(data) | |
| def partition_feature_names_by_group(fature_names: List[str]): | |
| """Partitions feature names into groups. | |
| Args: | |
| feature_names: List of feature names. | |
| Returns: | |
| Dictionary mapping feature groups to feature names and vice versa. | |
| """ | |
| feature_group_to_features_dict = collections.defaultdict(list) | |
| feature_to_feature_group_dict = {} | |
| for feat in fature_names: | |
| if 'SBS' in feat: | |
| feature_group_to_features_dict['signature'].append(feat) | |
| feature_to_feature_group_dict[feat] = 'signature' | |
| elif feat in ['Age', 'Sex']: | |
| feature_group_to_features_dict['clinical'].append(feat) | |
| feature_to_feature_group_dict[feat] = 'clinical' | |
| elif 'CNA' in feat: | |
| feature_group_to_features_dict['cna'].append(feat) | |
| feature_to_feature_group_dict[feat] = 'cna' | |
| else: | |
| feature_group_to_features_dict['mutation'].append(feat) | |
| feature_to_feature_group_dict[feat] = 'mutation' | |
| return feature_group_to_features_dict, feature_to_feature_group_dict | |
| def get_color(feature_name: str, feature_group_dict: Mapping[str, List[str]]) -> str: | |
| """ | |
| Determines the color for a given feature based on its group. | |
| Args: | |
| feature_name: Name of the feature. | |
| feature_group_dict: Dictionary mapping feature groups to feature names. | |
| Returns: | |
| A string representing the color associated with the feature's group. | |
| """ | |
| color_mapping = {'mutation': 'red', 'cna': 'green', 'signature': 'blue', 'clinical': 'grey'} | |
| for group, features in feature_group_dict.items(): | |
| if feature_name in features: | |
| return color_mapping[group] | |
| return 'black' # Default color if not found | |
| def get_individual_pred_interpretation(shap_pred_sample_df: pd.DataFrame, | |
| feature_sample_df: pd.DataFrame, | |
| feature_group_to_features_dict: dict, | |
| feature_to_feature_group_dict: dict, | |
| sample_info: str = None, | |
| filename: str = None, | |
| filepath: str = '../others_prediction_explanation', | |
| cohort_age_stats_path: str = '../data/combined_cohort_age_stats.pkl', | |
| top_feature_num: int = 10, | |
| save_plot: bool = False): | |
| """ | |
| Dynamic version of the function for individual prediction interpretation for a given tumor sample. | |
| Args: | |
| shap_pred_sample_df: DataFrame containing SHAP values for a given tumor sample. | |
| feature_sample_df: DataFrame containing feature values for a given tumor sample. | |
| feature_group_to_features_dict: Dictionary mapping feature groups to feature names. | |
| sample_info: Sample information to be displayed. | |
| filename: Filename to save the figure. | |
| top_feature_num: Number of top features to display. | |
| """ | |
| # Initialize plot | |
| plt.rcParams.update({'font.size': 15, "font.family": "Arial"}) | |
| fig, ax = plt.subplots() | |
| # remove top and right lines for bar graph | |
| ax.spines['right'].set_visible(False) | |
| ax.spines['top'].set_visible(False) | |
| # Sorting and selecting top features based on SHAP values | |
| all_features = sum(feature_group_to_features_dict.values(), []) | |
| sorted_features = sorted(all_features, key=lambda x: abs(shap_pred_sample_df.loc[x]), reverse=True) | |
| top_features = sorted(sorted_features[:top_feature_num], key=lambda x: shap_pred_sample_df.loc[x], reverse=False) | |
| # Preparing data for the bar chart | |
| top_feats_df = pd.DataFrame({ | |
| 'feat_name': top_features, | |
| 'SHAP_val': shap_pred_sample_df.loc[top_features], | |
| 'feat_val': feature_sample_df.loc[top_features], | |
| 'color': [get_color(feat, feature_group_to_features_dict) for feat in top_features] | |
| }) | |
| # Creating the bar chart | |
| ax.barh(top_feats_df['feat_name'], top_feats_df['SHAP_val'], color=top_feats_df['color']) | |
| ax.set_xlabel('SHAP Values') | |
| ax.set_title(sample_info) | |
| ax.set_yticks([]) | |
| with open(cohort_age_stats_path, "rb") as fp: | |
| combined_cohort_age_stats = pickle.load(fp) | |
| # Dynamic positioning of feature names and values | |
| left_margin = ax.get_xlim()[0] * 1.1 -.1 # Calculate the left margin dynamically | |
| for i, (name, value) in enumerate(zip(top_feats_df['feat_name'], top_feats_df['feat_val'])): | |
| value_text = f'{int(value)}' if feature_to_feature_group_dict[name] == 'mutation' else f'{value:.2f}' | |
| # Feature name and value | |
| if name == 'Sex': | |
| value_text = 'Male' if value == 1.0 else 'Female' | |
| if name == 'Age': | |
| value_text = int(value * combined_cohort_age_stats['Std_mean'] + combined_cohort_age_stats['Age_mean']) | |
| if name[-3:] == 'CNA': | |
| value_text = f'{int(value)}' | |
| ax.text(left_margin, i, f'{name}: {value_text}', ha='right', va='center', fontsize=10) | |
| ax.text(left_margin, top_feature_num, f'feature: value', ha='right', va='center', fontsize=10) | |
| # Adding legend | |
| legend_elements = [Patch(facecolor=color, label=label) for label, color in zip(['Somatic Mut.', 'CNA events', 'Mutation Sig.', 'Age/Sex'], ['red', 'green', 'blue', 'grey'])] | |
| ax.legend(handles=legend_elements, title='Feature Groups') | |
| plt.tight_layout() | |
| if save_plot: | |
| plt.savefig(f'{filepath}/{filename}.svg') | |
| plt.savefig(f'{filepath}/{filename}.pdf') | |
| plt.show() | |
| return f'{filepath}/{filename}.svg' | |
| def get_top_n_pred_and_shap( | |
| preds_df: pd.DataFrame, | |
| shaps: np.asarray, | |
| features: List[str], | |
| cancer_types: List[str], | |
| n: int=3) -> Mapping[str, Mapping[str, Any]]: | |
| """ | |
| Get top n predictions for each sample in the given DataFrame. | |
| Args: | |
| preds_df: DataFrame containing predictions. | |
| shaps: Array of SHAP values. | |
| features: List of features. | |
| n: Number of top predictions to return. | |
| cancer_types: List of cancer types. | |
| Returns: | |
| Dictionary containing top n predictions and SHAP values for each sample. | |
| """ | |
| # Filter only relevant columns (cancer types) | |
| df_chosen = preds_df[cancer_types] | |
| # Initialize the dictionary for SHAP values | |
| sample_id_to_top_n_pred_and_shaps_df = {} | |
| for sample_id, row in df_chosen.iterrows(): | |
| # Getting the top n predictions sorted by their probabilities | |
| sorted_top_3 = row.nlargest(n).sort_values(ascending=False) | |
| # Get row index for sample_id | |
| row_idx = preds_df.index.get_loc(sample_id) | |
| # Store the top n predictions and their probabilities | |
| shap_dfs = [] | |
| for pred_cancer in sorted_top_3.index: | |
| pred_cancer_idx = cancer_types.index(pred_cancer) | |
| shap_pred_cancer_df = pd.DataFrame(shaps[pred_cancer_idx][row_idx], | |
| index=features, | |
| columns=[pred_cancer]).T | |
| shap_dfs.append(shap_pred_cancer_df) | |
| shap_total_df = pd.concat(shap_dfs) | |
| sample_id_to_top_n_pred_and_shaps_df[sample_id] = { | |
| 'top_n_predictions': sorted_top_3.to_dict(), | |
| 'shap_values_df': shap_total_df | |
| } | |
| return sample_id_to_top_n_pred_and_shaps_df | |
| def get_onconpc_prediction_explanations(query_ids: List[str], | |
| preds_df: pd.DataFrame, | |
| shaps: np.array, | |
| df_features_genie: pd.DataFrame, | |
| cancer_types_to_consider: List[str], | |
| filepath: str='../others_prediction_explanation', | |
| cohort_age_stats_path: str='../data/combined_cohort_age_stats.pkl', | |
| save_plot: bool=False, | |
| ) -> List[Mapping[str, Any]]: | |
| """ | |
| Get OncoNPC predictions and generate SHAP-based explanation plots for multiple query IDs. | |
| Args: | |
| query_ids: List of IDs of the tumor samples to query. | |
| preds_df: DataFrame containing predictions. | |
| shaps: Array of SHAP values. | |
| df_features_genie: DataFrame containing features. | |
| cancer_types_to_consider: List of cancer types considered in the prediction. | |
| filepath: Path to save the explanation plots. | |
| save_plot: Whether to save the explanation plots. | |
| Returns: | |
| List of dictionaries containing prediction details and explanation plots for each query ID. | |
| """ | |
| results_dict = {} | |
| for query_id in query_ids: | |
| # Get OncoNPC prediction | |
| pred_prob = preds_df.at[query_id, 'max_posterior'] | |
| pred_cancer = preds_df.at[query_id, 'cancer_type'] | |
| pred_cancer_idx = cancer_types_to_consider.index(pred_cancer) | |
| # Get SHAP-based explanation for the prediction | |
| feature_sample_df = df_features_genie.loc[query_id] | |
| shap_pred_cancer_df = pd.DataFrame(shaps[pred_cancer_idx], | |
| index=df_features_genie.index, | |
| columns=df_features_genie.columns) | |
| shap_pred_sample_df = shap_pred_cancer_df.loc[query_id] | |
| # Information and plot generation | |
| sample_info = f'SAMPLE_ID: {query_id}\nPrediction: {pred_cancer}\nPrediction probability: {pred_prob:.3f}' | |
| feature_group_to_features_dict, feature_to_feature_group_dict = partition_feature_names_by_group(df_features_genie.columns) | |
| full_filename = get_individual_pred_interpretation(shap_pred_sample_df, | |
| feature_sample_df, | |
| feature_group_to_features_dict, | |
| feature_to_feature_group_dict, | |
| sample_info=sample_info, | |
| filename=str(query_id), | |
| filepath=filepath, | |
| top_feature_num=10, | |
| cohort_age_stats_path=cohort_age_stats_path, | |
| save_plot=save_plot) | |
| # Store the results | |
| results_dict[query_id] = { | |
| 'pred_prob': pred_prob, | |
| 'pred_cancer': pred_cancer, | |
| 'explanation_plot': full_filename | |
| } | |
| return results_dict | |
| def get_onconpc_features_from_raw_data(df_patients_chosen: pd.DataFrame, | |
| df_samples_chosen: pd.DataFrame, | |
| df_mutations_chosen: pd.DataFrame, | |
| df_cna_chosen: pd.DataFrame, | |
| features_onconpc_path: str='data/features_onconpc.pkl', | |
| combined_cohort_age_stats_path: str='data/combined_cohort_age_stats.pkl', | |
| mut_sig_weights_filepath: str='data/mutation_signatures/sigProfiler*.csv' | |
| ) -> Tuple[pd.DataFrame, pd.DataFrame]: | |
| """ | |
| Process raw GENIE data for OncoNPC prediction inference. | |
| Args: | |
| df_patients_chosen: DataFrame containing chosen patient data. | |
| df_samples_chosen: DataFrame containing chosen sample data. | |
| df_mutations_chosen: DataFrame containing chosen mutation data. | |
| df_cna_chosen: DataFrame containing chosen copy number alteration data. | |
| features_onconpc_path: Path to OncoNPC features file. | |
| combined_cohort_age_stats_path: Path to combined cohort age statistics file. | |
| weights_filepath: Path to mutation signatures weights file. | |
| Returns: | |
| Processed DataFrame ready for OncoNPC prediction inference. | |
| """ | |
| # Re-shape df_cna_chosen | |
| df_cna_reshaped = df_cna_chosen.set_index(['Hugo_Symbol']).T.copy() | |
| # Merge patient and sample data | |
| df_patients_merged = pd.merge(df_patients_chosen, df_samples_chosen, how='right', on='PATIENT_ID') | |
| # Process mutation data | |
| df_trinuc_feats_genie = get_snv_in_trinuc_context(df_mutations_chosen, | |
| sample_id_col='Tumor_Sample_Barcode', | |
| chromosome_col='Chromosome', | |
| start_pos_col='Start_Position', | |
| ref_allele_col='Reference_Allele', | |
| alt_allele_col='Tumor_Seq_Allele2', | |
| config='genie') | |
| # Obtain mutation signatures | |
| df_mut_sigs_genie = obtain_mutation_signatures(df_trinuc_feats_genie, weights_filepath=mut_sig_weights_filepath) | |
| # Preprocess features and labels | |
| df_features_genie, df_labels_genie = pre_process_features_genie(df_mutations_chosen, | |
| df_cna_reshaped, | |
| df_mut_sigs_genie, | |
| df_patients_merged) | |
| # Zero-pad missing features | |
| with open(features_onconpc_path, "rb") as fp: | |
| features_onconpc = pickle.load(fp) | |
| df_features_genie_final = zero_pad_missing_features(df_features_genie, features_onconpc) | |
| # Standardize feature names | |
| df_features_genie_final.columns = standardize_feat_names(df_features_genie_final.columns) | |
| # Standardize Age feature | |
| with open(combined_cohort_age_stats_path, "rb") as fp: | |
| combined_cohort_age_stats = pickle.load(fp) | |
| df_features_genie_final['Age'] = (df_features_genie_final['Age'] - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean'] | |
| return df_features_genie_final, df_labels_genie | |
| def get_snv_in_trinuc_context(df_mutations: pd.DataFrame, | |
| sample_id_col: str, | |
| chromosome_col: str, | |
| start_pos_col: str, | |
| ref_allele_col: str, | |
| alt_allele_col: str, | |
| config: Optional[str]='genie') -> pd.DataFrame: | |
| """ | |
| Processes the given mutations DataFrame to get SNV in trinucleotide context. | |
| Args: | |
| df_mutations: DataFrame containing mutation data. | |
| sample_id_col: Column name for sample IDs. | |
| chromosome_col: Column name for chromosome. | |
| start_pos_col: Column name for start position. | |
| ref_allele_col: Column name for reference allele. | |
| alt_allele_col: Column name for alternative allele. | |
| Returns: | |
| DataFrame with SNV in trinucleotide context. | |
| """ | |
| # Specify the columns to be chosen | |
| columns_chosen = [sample_id_col, chromosome_col, start_pos_col, ref_allele_col, alt_allele_col] | |
| # Process the DataFrame | |
| df_mutations_chosen_trinuc = df_mutations[columns_chosen].copy() | |
| if config == 'genie': | |
| df_mutations_chosen_trinuc[chromosome_col] = 'chr' + df_mutations_chosen_trinuc[chromosome_col].astype(str) | |
| df_mutations_chosen_trinuc = df_mutations_chosen_trinuc[~df_mutations_chosen_trinuc[chromosome_col].str.contains('GL|chrMT')] | |
| # Change the data types for R processing | |
| df_mutations_chosen_trinuc[sample_id_col] = df_mutations_chosen_trinuc[sample_id_col].astype('str') | |
| # R function code | |
| r_code = ''' | |
| function(mutationData, sample_id, chr, pos, ref, alt) { | |
| library(deconstructSigs) | |
| sigs.input <- mut.to.sigs.input(mut.ref = mutationData, | |
| sample.id = sample_id, | |
| chr = chr, | |
| pos = pos, | |
| ref = ref, | |
| alt = alt) | |
| # Filter samples with low mutations | |
| sigs.input <- sigs.input[rowSums(sigs.input) >= 1,] | |
| return(sigs.input) | |
| } | |
| ''' | |
| # Convert the DataFrame for R processing | |
| with conversion.localconverter(default_converter): | |
| with localconverter(robjects.default_converter + pandas2ri.converter): | |
| r_df_mutations = robjects.conversion.py2rpy(df_mutations_chosen_trinuc) | |
| # Load and call the R function | |
| r_function = robjects.r(r_code) | |
| with conversion.localconverter(default_converter): | |
| with localconverter(robjects.default_converter + pandas2ri.converter): | |
| df_trinuc_feats = robjects.conversion.rpy2py(r_function(r_df_mutations, | |
| sample_id_col, | |
| chromosome_col, | |
| start_pos_col, | |
| ref_allele_col, | |
| alt_allele_col)) | |
| if config=='profile_dfci': | |
| df_trinuc_feats.index = df_trinuc_feats.index.astype('float') | |
| return df_trinuc_feats | |
| def obtain_mutation_signatures(df_trinuc_feats: pd.DataFrame, | |
| weights_filepath: str='../data/mutation_signatures/sigProfiler*.csv' | |
| ) -> pd.DataFrame: | |
| """Transforms tri-nucleotide features into mutation signature based features. | |
| Args: | |
| df_trinuc_feats: DataFrame containing tri-nucleotide features. | |
| Returns: | |
| mut_sig_based_df: DataFrame containing mutation signature based features. | |
| """ | |
| # Load Mutational Signatures COSMIC data. | |
| file_names = glob.glob(weights_filepath) | |
| # Check if the directory exists. | |
| if len(file_names) == 0: | |
| raise ValueError('No mutation signatures data.') | |
| file_names_key = [file_name.split('_')[-1][:-4] for file_name in file_names] | |
| sbs_conversion_df_dict = {} | |
| for i in range(len(file_names)): | |
| df = pd.read_csv(file_names[i]) | |
| df.dropna(inplace = True) | |
| # process base names s.t. they compatible with actual feature names | |
| new_name_list = [] | |
| for change, subtype in zip(df.Type.values, df.Subtype.values): | |
| new_name = subtype[0] + '[' + change + ']' + subtype[2] | |
| new_name_list.append(new_name) | |
| df['Type_new'] = new_name_list | |
| df.set_index('Type_new', inplace = True) | |
| # check if column contains human-based experiment results | |
| merged_colum_names = ''.join(df.columns) | |
| if 'GRCh38' in merged_colum_names: | |
| df['values_oi'] = df[file_names_key[i] + '_GRCh38'].values | |
| elif 'GRCh37' in merged_colum_names: | |
| df['values_oi'] = df[file_names_key[i] + '_GRCh37'].values | |
| else: | |
| raise ValueError('No human-based experimental results.') | |
| sum_vals = np.sum(df.values_oi.values) | |
| if abs(sum_vals - 1.0) > 0.1: | |
| raise ValueError('Values do not sum up to 1.') | |
| sbs_conversion_df_dict[file_names_key[i]] = df | |
| mut_sig_based_df = pd.DataFrame([], index = df_trinuc_feats.index) | |
| for key, mut_df in sbs_conversion_df_dict.items(): | |
| common_substitute_feats = list(set(mut_df.index) & set(df_trinuc_feats.columns)) | |
| if len(common_substitute_feats) != 96: | |
| raise ValueError('Number of substitution features does not sum up to 96.') | |
| # Get mutation signature values | |
| mut_sig_vals = np.matmul(df_trinuc_feats[common_substitute_feats].values, | |
| mut_df.loc[common_substitute_feats].values_oi.values) | |
| mut_sig_based_df[key] = mut_sig_vals | |
| return mut_sig_based_df | |
| def pre_process_features_genie(df_mutations: pd.DataFrame, | |
| df_cna: pd.DataFrame, | |
| df_mutation_signatures: pd.DataFrame, | |
| df_patients: pd.DataFrame, | |
| cancer_types: Optional[List[str]]=None, | |
| id_column: str = 'SAMPLE_ID', | |
| cancer_type_column: str='CANCER_TYPE') -> Tuple[pd.DataFrame, pd.DataFrame]: | |
| """Pre-process genetics data to create feature df for GENIE. | |
| Args: | |
| df_mutations: DataFrame containing mutation data. | |
| df_cna: DataFrame containing CNA data. | |
| df_mutation_signatures: DataFrame containing mutation signature data. | |
| df_patients: DataFrame containing patient data. | |
| cancer_types: List of cancer types. | |
| id_column: Column name for sample IDs. | |
| cancer_type_column: Column name for cancer types. | |
| Returns: | |
| df_features_merged_final: DataFrame containing merged features. | |
| df_labels_final: DataFrame containing labels. | |
| """ | |
| # Get mutation features. | |
| sample_ids = df_patients[id_column].values | |
| df_mutations_chosen = df_mutations.loc[df_mutations.Tumor_Sample_Barcode.isin(sample_ids)] | |
| mutation_gene_names = np.unique(df_mutations_chosen.Hugo_Symbol.values) | |
| df_mutation_feature = pd.DataFrame(0, columns=mutation_gene_names, index=sample_ids) | |
| for entry in df_mutations_chosen[['Tumor_Sample_Barcode', 'Hugo_Symbol']].values: | |
| sample_id = entry[0] | |
| gene = entry[1] | |
| if gene in df_mutation_feature.columns: | |
| # Count the number of mutation frequency in each gene. | |
| df_mutation_feature.at[sample_id, gene] = df_mutation_feature.at[sample_id, gene] + 1 | |
| # Add 'mut' at the end of each mutation feature | |
| df_mutation_feature.columns = [gene + '_mut' for gene in mutation_gene_names] | |
| # Get CNA features. | |
| df_cna_chosen = df_cna.loc[list(set(df_cna.index) & set(sample_ids))] | |
| df_cna_chosen.fillna(0, inplace = True) | |
| # Obtain age/sex features. | |
| df_patients_id_indexed = df_patients.set_index('SAMPLE_ID') | |
| df_sex_age_feature = df_patients_id_indexed[['SEX', 'AGE_AT_SEQ_REPORT']].copy() | |
| sex_list = [] | |
| age_nan_indices = [] | |
| age_list = [] | |
| for sample_id, (sex, age) in zip(df_sex_age_feature.index, df_sex_age_feature.values): | |
| if sex == 'Male': | |
| sex_list.append(1) | |
| elif sex == 'Female': | |
| sex_list.append(-1) | |
| else: # Not reported | |
| sex_list.append(0) | |
| if pd.isnull(age): | |
| age_nan_indices.append(sample_id) | |
| age_list.append(0) # append zero for now | |
| elif '>89' in str(age): | |
| age_list.append(int(age[1:])) | |
| elif '<18' in str(age): | |
| age_list.append(int(age[1:])) | |
| else: | |
| age_list.append(int(age)) | |
| df_sex_age_feature['SEX'] = sex_list | |
| df_sex_age_feature['AGE_AT_SEQ_REPORT'] = age_list | |
| df_sex_age_feature.columns = ['Sex', 'Age'] | |
| # Merge all feature dfs using common sample IDs. | |
| common_ids = list(set(df_mutation_feature.index) & set(df_cna_chosen.index) & set(df_mutation_signatures.index) | |
| & set(df_sex_age_feature.index)) | |
| dfs_list = [df_mutation_feature, df_cna_chosen, df_mutation_signatures, df_sex_age_feature] | |
| df_features_merged = merge_dfs(dfs_list, common_ids) | |
| # Exclude age NaN indices. | |
| df_features_merged_final = df_features_merged.loc[list(set(df_features_merged.index) - set(age_nan_indices))] | |
| df_labels = df_patients_id_indexed.loc[df_features_merged_final.index][cancer_type_column] | |
| if cancer_types is not None: | |
| df_labels_final = pd.DataFrame([cancer_types.index(val) for val in df_labels.values], | |
| columns = ['cancer_label'], index = df_labels.index) | |
| df_labels_final['cancer_type'] = df_labels.values | |
| return df_features_merged_final, df_labels_final | |
| else: | |
| return df_features_merged_final, df_labels | |
| def pre_process_features_dfci(df_mutations: pd.DataFrame, | |
| df_cna: pd.DataFrame, | |
| df_mutation_signatures: pd.DataFrame, | |
| df_patients: pd.DataFrame, | |
| cancer_types: List[str], | |
| cup_samples_ids: Optional[List[str]]=None, | |
| id_column: str = 'UNIQUE_SAMPLE_ID' | |
| ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: | |
| """Pre-process genetics data to create feature df for DFCI data. | |
| Args: | |
| df_mutations: DataFrame containing mutation data. | |
| df_cna: DataFrame containing CNA data. | |
| df_mutation_signatures: DataFrame containing mutation signature data. | |
| df_patients: DataFrame containing patient data. | |
| cancer_types: List of cancer types. | |
| cup_samples_ids: List of CUP sample IDs. | |
| id_column: Column name for sample IDs. | |
| Returns: | |
| df_features_merged_ckps: DataFrame containing merged features for CKP samples. | |
| df_labels_final: DataFrame containing labels for CKP samples. | |
| df_features_merged_cups: DataFrame containing merged features for CUP samples. | |
| """ | |
| # Get mutation features. | |
| sample_ids = df_patients[id_column].values | |
| df_mutations_chosen = df_mutations.loc[df_mutations[id_column].isin(sample_ids)] | |
| mutation_gene_names = np.unique(df_mutations_chosen.CANONICAL_GENE.values) | |
| df_mutation_feature = pd.DataFrame(0, columns=mutation_gene_names, index=sample_ids) | |
| for entry in df_mutations_chosen[['UNIQUE_SAMPLE_ID', 'CANONICAL_GENE']].values: | |
| sample_id = entry[0]; gene = entry[1]; | |
| if gene in df_mutation_feature.columns: | |
| # Count the number of mutation frequency in each gene. | |
| df_mutation_feature.at[sample_id, gene] = df_mutation_feature.at[sample_id, gene] + 1 | |
| # Add 'mut' at the end of each mutation feature | |
| df_mutation_feature.columns = [gene + '_mut' for gene in mutation_gene_names] | |
| # Get CNA features. | |
| df_cna_chosen = df_cna.loc[list(set(df_cna.index) & set(sample_ids))] | |
| df_cna_chosen.fillna(0, inplace = True) | |
| # Obtain age/sex features. | |
| df_patients_id_indexed = df_patients.set_index(id_column) | |
| df_sex_age_feature = df_patients_id_indexed[['Sex', 'Age']].copy() | |
| sex_list = [] | |
| for sample_id, (sex, age) in zip(df_sex_age_feature.index, df_sex_age_feature.values): | |
| if sex == 'MALE': | |
| sex_list.append(1) | |
| elif sex == 'FEMALE': | |
| sex_list.append(-1) | |
| else: # Not reported | |
| sex_list.append(0) | |
| df_sex_age_feature['Sex'] = sex_list | |
| df_sex_age_feature.columns = ['Sex', 'Age'] | |
| # Merge all features using the same sample IDs. | |
| common_ids = list(set(df_mutation_feature.index) & set(df_cna_chosen.index) & | |
| set(df_mutation_signatures.index) & set(df_sex_age_feature.index)) | |
| dfs_list = [df_mutation_feature, df_cna_chosen, df_mutation_signatures, df_sex_age_feature] | |
| df_features_merged = merge_dfs(dfs_list, common_ids) | |
| # Make gene names consistent across GENIE and PROFILE. | |
| profile_old_new_gene_mapping = {} | |
| profile_old_new_gene_mapping['C17ORF70'] = 'FAAP100' | |
| profile_old_new_gene_mapping['C17orf70_mut'] = 'FAAP100_mut' | |
| profile_old_new_gene_mapping['C19ORF40'] = 'FAAP24' | |
| profile_old_new_gene_mapping['C19orf40_mut'] = 'FAAP24_mut' | |
| profile_old_new_gene_mapping['LOC96610_mut'] = 'BMS1P20_mut' | |
| profile_old_new_gene_mapping['LOC729991-MEF2B_mut'] = 'MEF2BNB-MEF2B_mut' | |
| profile_old_new_gene_mapping['C1orf86_mut'] = 'FAAP20_mut' | |
| profile_old_new_gene_mapping['C1ORF86'] = 'FAAP20' | |
| profile_old_new_gene_mapping['GNB2L1_mut'] = 'RACK1_mut' | |
| profile_old_new_gene_mapping['GNB2L1'] = 'RACK1' | |
| df_features_merged.rename(columns=profile_old_new_gene_mapping, | |
| inplace=True) | |
| df_features_merged_ckps = (df_features_merged | |
| .loc[list(set(df_features_merged.index) - set(cup_samples_ids))]) | |
| df_features_merged_cups = (df_features_merged | |
| .loc[list(set(df_features_merged.index) & set(cup_samples_ids))]) | |
| df_labels = df_patients_id_indexed.loc[df_features_merged_ckps.index]['CANCER_TYPE'] | |
| df_labels_final = pd.DataFrame([cancer_types.index(val) for val in df_labels.values], | |
| columns=['cancer_label'], index=df_labels.index) | |
| df_labels_final['cancer_type'] = df_labels.values | |
| return df_features_merged_ckps, df_labels_final, df_features_merged_cups | |
| def zero_pad_missing_features(df: pd.DataFrame, | |
| feature_list: List[str]) -> pd.DataFrame: | |
| """ | |
| Zero pad missing features. | |
| Args: | |
| df: DataFrame containing features. | |
| feature_list: List of features to include. | |
| Returns: | |
| df: DataFrame containing zero-padded features. | |
| """ | |
| # Create a dictionary for new columns to add | |
| new_columns = {feature: [0] * len(df) for feature in feature_list if feature not in df.columns} | |
| # Concatenate new columns with the original DataFrame | |
| df = pd.concat([df, pd.DataFrame(new_columns, index=df.index)], axis=1) | |
| return df[feature_list] | |
| def merge_dfs(df_list: pd.DataFrame, ids_common: List[str]) -> pd.DataFrame: | |
| """Merges a list of dataframes using common sample IDs. | |
| Args: | |
| df_list: List of dataframes to merge. | |
| ids_common: Common sample IDs. | |
| Returns: | |
| df_merged_to_return: Merged dataframe. | |
| """ | |
| for idx, df in enumerate(df_list): | |
| if idx == 0: | |
| df_merged_to_return = df.loc[ids_common] | |
| else: | |
| df_merged_to_return = pd.merge(df_merged_to_return, df.loc[ids_common], how = 'left', left_index = True, right_index = True) | |
| return df_merged_to_return | |
| def standardize_feat_names(curr_feat_names: List[str]) -> List[str]: | |
| """ | |
| Standardize feature names. | |
| Args: | |
| curr_feat_names: list of feature names | |
| Returns: | |
| new_feat_names: list of standardized feature names | |
| """ | |
| new_feat_names = [] | |
| for feat in curr_feat_names: | |
| if '_mut' in feat: | |
| new_feat_names.append(feat.replace('_mut', '')) | |
| elif 'AGE' in feat or 'Age' in feat: | |
| new_feat_names.append('Age') | |
| elif 'GENDER' in feat or 'Sex' in feat: | |
| new_feat_names.append('Sex') | |
| elif 'SBS' in feat: | |
| new_feat_names.append(feat) | |
| else: | |
| new_feat_names.append(feat + ' CNA') | |
| return new_feat_names |