Spaces:
Sleeping
Sleeping
File size: 8,721 Bytes
436f829 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | from absl import app
from absl import flags
import pandas as pd
import pickle
import utils_training
import utils
import os
import numpy as np
"""
Author: Intae Moon
In this script, we train and evaluate the performance of the XGBoost model on the OncoTree-based cancer types.
One can choose to use k-fold cross validation or train on the entire dataset and test on the held out set.
"""
flags.DEFINE_string('config', None, 'Cancer centers to incorporate for feature processing (genie, profile_dfci, or both)')
flags.DEFINE_integer('k_fold', 10, 'Number of folds in k-fold cross-validation')
flags.DEFINE_boolean('use_held_out_set', False, 'Whether to use a held-out set')
flags.DEFINE_boolean('filter_out_non_informatives', False, 'Whether to filter out non-informative features and samples')
flags.DEFINE_string('save_model_name', None, 'Name of the model to save')
flags.DEFINE_string('index_profile_samples_by', 'SAMPLE_ACCESSION_NBR', 'Index profile samples by DFCI_MRN or SAMPLE_ACCESSION_NBR')
# DATA_PATH = '../data/internal_training_data/'
DATA_PATH = '../data/'
def main(argv=None):
# Custom check for flags that you still require to be non-default
FLAGS = flags.FLAGS
# Training configs:
config = FLAGS.config
k_fold = FLAGS.k_fold
use_held_out_set = FLAGS.use_held_out_set
save_model_name = FLAGS.save_model_name
index_profile_samples_by = FLAGS.index_profile_samples_by
filter_out_non_informatives = FLAGS.filter_out_non_informatives
if config not in ['genie', 'profile_dfci', 'both']:
raise ValueError("config must be one of genie, profile_dfci, or both")
if config == 'genie' and use_held_out_set == True:
raise ValueError("Cannot use held out set when training on genie data only")
# load xgb parameters
hyperparam_xgb = pd.read_csv(os.path.join(DATA_PATH, 'xgb_hyperparams'), sep = '\t') #xgb_params_to_run # params_w_results_Mar_1st
hyperparam_xgb.drop(columns = 'Unnamed: 0', inplace = True)
# Get the best performing hyperparameters from the previous experiments
# this can be extended to try multiple hyperparameters
params_xgb = hyperparam_xgb.iloc[0:1]
# ====================================================================================================
# Loading data and labels, and processing them for training
# ====================================================================================================
# Load trainable data and labels
# Tab separated feature data for CKPs
feature_data_name = os.path.join(DATA_PATH, 'features_genie_') # Replace with your feature file path
# Tab separated label data for CKPs
label_data_name = os.path.join(DATA_PATH, 'labels_genie_') # Replace with your label file path
if config in ['profile_dfci', 'both']:
# Tab separated feature data for CUPs
feature_data_name_cup = os.path.join(DATA_PATH, 'features_combined_cup_onco_tree_based_dev')
features_cup_df = pd.read_csv(feature_data_name_cup, sep = '\t')
features_cup_df.set_index('Unnamed: 0', inplace = True)
features_ckp_df = pd.read_csv(feature_data_name, sep = '\t')
features_ckp_df.set_index(features_ckp_df.columns[0], inplace = True)
labels_ckp_df = pd.read_csv(label_data_name, sep = '\t')
labels_ckp_df.set_index(labels_ckp_df.columns[0], inplace = True)
if config in ['profile_dfci', 'both']:
if index_profile_samples_by == 'DFCI_MRN':
# Load ../data/internal_training_data/onconpc_sample_id_to_dfci_mrn.pkl
with open(os.path.join(DATA_PATH, 'onconpc_sample_id_to_dfci_mrn.pkl'), "rb") as fp: # Unpickling
onconpc_sample_id_to_dfci_mrn = pickle.load(fp)
# index CKP data by DFCI_MRN
features_ckp_df.index = utils_training.get_new_indices(features_ckp_df.index, onconpc_sample_id_to_dfci_mrn)
labels_ckp_df.index = utils_training.get_new_indices(labels_ckp_df.index, onconpc_sample_id_to_dfci_mrn)
features_cup_df.index = utils_training.get_new_indices(features_cup_df.index, onconpc_sample_id_to_dfci_mrn)
# Set index name
features_ckp_df.index.name = 'DFCI_MRN_FOR_PROFILE'
labels_ckp_df.index.name = 'DFCI_MRN_FOR_PROFILE'
features_cup_df.index.name = 'DFCI_MRN_FOR_PROFILE'
elif index_profile_samples_by == 'SAMPLE_ACCESSION_NBR':
# Load ../data/internal_training_data/onconpc_sample_id_to_sample_accession_nbr.pkl
with open(os.path.join(DATA_PATH, 'onconpc_sample_id_to_sample_accession_nbr.pkl'), "rb") as fp: # Unpickling
onconpc_sample_id_to_sample_accession_nbr = pickle.load(fp)
# index CKP data by SAMPLE_ACCESSION_NBR
features_ckp_df.index = utils_training.get_new_indices(features_ckp_df.index, onconpc_sample_id_to_sample_accession_nbr)
labels_ckp_df.index = utils_training.get_new_indices(labels_ckp_df.index, onconpc_sample_id_to_sample_accession_nbr)
features_cup_df.index = utils_training.get_new_indices(features_cup_df.index, onconpc_sample_id_to_sample_accession_nbr)
# Set index name
features_ckp_df.index.name = 'SAMPLE_ACCESSION_NBR_FOR_PROFILE'
labels_ckp_df.index.name = 'SAMPLE_ACCESSION_NBR_FOR_PROFILE'
features_cup_df.index.name = 'SAMPLE_ACCESSION_NBR_FOR_PROFILE'
if use_held_out_set:
heldout_ckps_preds_df = pd.read_csv(os.path.join(DATA_PATH, 'heldout_ckps_preds_onco_tree_based_dev'), sep = '\t')
heldout_ckps_preds_df.set_index(heldout_ckps_preds_df.columns[0], inplace = True)
held_out_indices = utils_training.get_new_indices([idx[:-3] for idx in heldout_ckps_preds_df.index],
onconpc_sample_id_to_sample_accession_nbr)
# Exclude held out samples from training
indices_to_choose = set(features_ckp_df.index) - set(held_out_indices)
features_ckp_df = features_ckp_df.loc[indices_to_choose]
labels_ckp_df = labels_ckp_df.loc[indices_to_choose]
feature_group_to_features_dict = utils.partition_feature_names_by_group(list(features_ckp_df.columns))
if filter_out_non_informatives:
if config in ['profile_dfci', 'both']:
(features_ckp_final_df,
labels_ckp_final_df,
features_cup_final_df) = utils_training.filter_out_low_freq_feats_and_samples(features_ckp_df,
labels_ckp_df,
features_cup_df,
feature_group_to_features_dict)
else:
(features_ckp_final_df,
labels_ckp_final_df,
_) = utils_training.filter_out_low_freq_feats_and_samples(features_ckp_df,
labels_ckp_df,
features_ckp_df, # placeholder for cup_df
feature_group_to_features_dict)
else:
features_ckp_final_df = features_ckp_df
labels_ckp_final_df = labels_ckp_df
if config in ['profile_dfci', 'both']:
features_cup_final_df = features_cup_df
# ====================================================================================================
# Model Training and Evaluation
# ====================================================================================================
# Cancer types to consider
cancer_types = list(np.unique(labels_ckp_final_df.cancer_type.values))
# Check duplicate indices
if len(features_ckp_final_df) == len(set(features_ckp_final_df.index)):
print('\n')
print('No duplicate indices')
else:
print('\n')
print('Duplicate indices detected (most likely due to DFCI_MRN indexing)')
if k_fold > 0:
(k_fold_to_performance_report_dict,
pred_probs_on_val_total_df) = utils_training.perform_k_fold(features_ckp_final_df,
labels_ckp_final_df,
cancer_types,
params_xgb,
k_fold=k_fold,
save_model_name=save_model_name)
# Store k_fold_to_performance_report_dict and pred_probs_on_val_total_df
with open(os.path.join(DATA_PATH, f'k_fold_to_performance_report_dict_{save_model_name}.pkl'), 'wb') as f:
pickle.dump(k_fold_to_performance_report_dict, f)
pred_probs_on_val_total_df.to_csv(os.path.join(DATA_PATH, f'pred_probs_on_val_total_df_{save_model_name}.csv'))
else:
X_train = features_ckp_final_df.copy()
y_train = labels_ckp_final_df.loc[X_train.index]['cancer_label']
# Standardize Age based on train data
age_mean = X_train['Age'].mean()
age_std = X_train['Age'].std()
X_train['Age'] = (X_train['Age'] - age_mean) / age_std
xg_clf = utils_training.fit_and_evaluate_model(X_train.values,
y_train.values,
None,
None,
params_xgb,
cancer_types)
# Evaluate the model performance based on different maximum prediction probability cut-offs.
if save_model_name is not None:
if not os.path.exists('../models'):
os.makedirs('../models/models')
xg_clf.save_model(f'../models/xgboost_{save_model_name}_trained_on_all_ckps.json')
return
if __name__ == '__main__':
flags.mark_flags_as_required(['k_fold', 'config', 'use_held_out_set', 'save_model_name'])
app.run(main)
|