Spaces:
Sleeping
Sleeping
Commit ·
30e4166
1
Parent(s): 0f4d482
file paths
Browse files- codes/gradio_utils.py +9 -71
- codes/process_features.py +11 -11
- codes/utils.py +5 -5
- codes/utils_training.py +1 -1
codes/gradio_utils.py
CHANGED
|
@@ -49,7 +49,7 @@ cancer_types_to_consider = ['Acute Myeloid Leukemia', 'Bladder Urothelial Carcin
|
|
| 49 |
'Pancreatic Neuroendocrine Tumor', 'Pleural Mesothelioma', 'Prostate Adenocarcinoma', 'Renal Cell Carcinoma',
|
| 50 |
'Well-Differentiated Thyroid Cancer']
|
| 51 |
|
| 52 |
-
all_features = pd.read_csv('.
|
| 53 |
|
| 54 |
def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
|
| 55 |
|
|
@@ -140,17 +140,17 @@ def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
|
|
| 140 |
samples_df,
|
| 141 |
mutations_df,
|
| 142 |
cna_df,
|
| 143 |
-
features_onconpc_path='.
|
| 144 |
-
combined_cohort_age_stats_path='.
|
| 145 |
-
mut_sig_weights_filepath='.
|
| 146 |
)
|
| 147 |
-
df_features_genie_final.to_csv('.
|
| 148 |
sample_id = tumor_id
|
| 149 |
features = df_features_genie_final
|
| 150 |
|
| 151 |
# load fully trained OncoNPC model
|
| 152 |
xgb_onconpc = xgb.Booster()
|
| 153 |
-
xgb_onconpc.load_model('.
|
| 154 |
|
| 155 |
# predict primary sites of CUP tumors
|
| 156 |
predictions = utils.get_xgboost_latest_cancer_type_preds(xgb_onconpc,
|
|
@@ -173,67 +173,6 @@ def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
|
|
| 173 |
|
| 174 |
return get_top3(predictions, tumor_id), gr.Markdown(markdown_text), results[tumor_id]['explanation_plot']
|
| 175 |
|
| 176 |
-
# def parse_inputs(age, gender, CNA_events, mutations, sample_id):
|
| 177 |
-
|
| 178 |
-
# # Normalization of age input
|
| 179 |
-
# combined_cohort_age_stats = None
|
| 180 |
-
# combined_cohort_age_stats_path = '../data/combined_cohort_age_stats.pkl'
|
| 181 |
-
# with open(combined_cohort_age_stats_path, "rb") as fp:
|
| 182 |
-
# combined_cohort_age_stats = pickle.load(fp)
|
| 183 |
-
# age = (age - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean']
|
| 184 |
-
|
| 185 |
-
# # Initialize data dictionary and populate with age, sex, and sample id
|
| 186 |
-
# data = {'AGE_AT_SEQ_REPORT': age, 'SEX':gender, 'SAMPLE_ID':sample_id}
|
| 187 |
-
# df_patients = pd.DataFrame([data])
|
| 188 |
-
|
| 189 |
-
# # Process CNA events: Expected format [[CNA, val], [CNA, val]]
|
| 190 |
-
# if len(CNA_events) > 0:
|
| 191 |
-
# CNA_events = CNA_events.split('|')
|
| 192 |
-
# for i in range(len(CNA_events)):
|
| 193 |
-
# # Split each event into CNA and value, and cast the value to integer
|
| 194 |
-
# CNA, val = CNA_events[i].split()
|
| 195 |
-
# CNA_events[i] = [CNA + ' CNA', int(val)] # Cast val to integer
|
| 196 |
-
# else:
|
| 197 |
-
# CNA_events = []
|
| 198 |
-
|
| 199 |
-
# CNA_dict = {}
|
| 200 |
-
# # Add CNA events to data dictionary
|
| 201 |
-
# for CNA, val in CNA_events:
|
| 202 |
-
# CNA_dict[CNA] = val
|
| 203 |
-
|
| 204 |
-
# df_cna = pd.DataFrame([CNA_dict])
|
| 205 |
-
|
| 206 |
-
# # Process mutations: Expected format [mut1, mut2, etc.]
|
| 207 |
-
# if len(mutations) > 0:
|
| 208 |
-
# mutations = mutations.split('| ')
|
| 209 |
-
# for i in range(len(mutations)):
|
| 210 |
-
# # Split each mutation entry and strip white space
|
| 211 |
-
# mutations[i] = ['manual input'] + mutations[i].split(', ')
|
| 212 |
-
# mutations[i] = [m.strip() for m in mutations[i]] # Strip white space
|
| 213 |
-
# else:
|
| 214 |
-
# mutations = []
|
| 215 |
-
|
| 216 |
-
# # Define mutation columns for DataFrame and create mutation DataFrame
|
| 217 |
-
# mutation_columns = ["UNIQUE_SAMPLE_ID", "HUGO_SYMBOL", "CHROMOSOME", "POSITION", "REF_ALLELE", "ALT_ALLELE"]
|
| 218 |
-
# mutation_full_df = pd.DataFrame(mutations, columns=mutation_columns) if mutations else pd.DataFrame(columns=mutation_columns)
|
| 219 |
-
# mutation_partial_df = mutation_full_df[['UNIQUE_SAMPLE_ID', 'HUGO_SYMBOL']]
|
| 220 |
-
# mutation_partial_df = mutation_partial_df.rename(columns={'UNIQUE_SAMPLE_ID':'Tumor_Sample_Barcode', 'HUGO_SYMBOL':'Hugo_Symbol'})
|
| 221 |
-
# mutation_df = mutation_full_df.drop('HUGO_SYMBOL', axis=1, inplace=False)
|
| 222 |
-
|
| 223 |
-
# # Save mutation data to a CSV file
|
| 224 |
-
# mutation_df.to_csv('./mutation_input.csv', index=False)
|
| 225 |
-
|
| 226 |
-
# # Get base substitution file and read into DataFrame
|
| 227 |
-
# base_sub_file = deconstructSigs.get_base_substitutions()
|
| 228 |
-
# df_trinuc_feats = pd.read_csv(base_sub_file)
|
| 229 |
-
# # Obtain mutation signatures
|
| 230 |
-
# mutation_signatures = utils.obtain_mutation_signatures(df_trinuc_feats)
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
# # Return processed features
|
| 234 |
-
# return utils.pre_process_features_genie(mutation_partial_df, df_cna, mutation_signatures, df_patients)[0] # just features, not labels
|
| 235 |
-
|
| 236 |
-
|
| 237 |
def parse_inputs(age, gender, CNA_events, mutations):
|
| 238 |
# Normalization of age input
|
| 239 |
age = age
|
|
@@ -242,7 +181,7 @@ def parse_inputs(age, gender, CNA_events, mutations):
|
|
| 242 |
except:
|
| 243 |
raise gr.Error('Age should be a positive numerical value')
|
| 244 |
combined_cohort_age_stats = None
|
| 245 |
-
combined_cohort_age_stats_path = '.
|
| 246 |
with open(combined_cohort_age_stats_path, "rb") as fp:
|
| 247 |
combined_cohort_age_stats = pickle.load(fp)
|
| 248 |
age = (age - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean']
|
|
@@ -338,8 +277,7 @@ def get_preds_min_info(age, gender, CNA_events, mutations, output='Top Predictio
|
|
| 338 |
|
| 339 |
# Parse input features
|
| 340 |
features = parse_inputs(age, gender, CNA_events, mutations)
|
| 341 |
-
|
| 342 |
-
all_features = pd.read_csv('../data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
|
| 343 |
columns_to_add = {column: [0] * len(features) for column in all_features if column not in features.columns}
|
| 344 |
print(columns_to_add)
|
| 345 |
for column in features.columns:
|
|
@@ -444,7 +382,7 @@ def update_image(target):
|
|
| 444 |
# Generate explanation plot
|
| 445 |
sample_info = f'Prediction: {target}\nPrediction probability: {probability:.3f}'
|
| 446 |
feature_group_to_features_dict, feature_to_feature_group_dict = utils.partition_feature_names_by_group(features.columns)
|
| 447 |
-
explanation_plot = utils.get_individual_pred_interpretation(shap_pred_sample_df, feature_sample_df, feature_group_to_features_dict, feature_to_feature_group_dict,sample_info=sample_info, filename=f'{sample_id}_{target}_plot', filepath='.
|
| 448 |
return explanation_plot
|
| 449 |
|
| 450 |
def show_row(value):
|
|
|
|
| 49 |
'Pancreatic Neuroendocrine Tumor', 'Pleural Mesothelioma', 'Prostate Adenocarcinoma', 'Renal Cell Carcinoma',
|
| 50 |
'Well-Differentiated Thyroid Cancer']
|
| 51 |
|
| 52 |
+
all_features = pd.read_csv('./data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
|
| 53 |
|
| 54 |
def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
|
| 55 |
|
|
|
|
| 140 |
samples_df,
|
| 141 |
mutations_df,
|
| 142 |
cna_df,
|
| 143 |
+
features_onconpc_path='./data/features_onconpc.pkl',
|
| 144 |
+
combined_cohort_age_stats_path='./data/combined_cohort_age_stats.pkl',
|
| 145 |
+
mut_sig_weights_filepath='./data/mutation_signatures/sigProfiler*.csv'
|
| 146 |
)
|
| 147 |
+
df_features_genie_final.to_csv('./data/onconpc_features2.csv', index=True)
|
| 148 |
sample_id = tumor_id
|
| 149 |
features = df_features_genie_final
|
| 150 |
|
| 151 |
# load fully trained OncoNPC model
|
| 152 |
xgb_onconpc = xgb.Booster()
|
| 153 |
+
xgb_onconpc.load_model('./model/xgboost_v1.7.6_OncoNPC_full.json')
|
| 154 |
|
| 155 |
# predict primary sites of CUP tumors
|
| 156 |
predictions = utils.get_xgboost_latest_cancer_type_preds(xgb_onconpc,
|
|
|
|
| 173 |
|
| 174 |
return get_top3(predictions, tumor_id), gr.Markdown(markdown_text), results[tumor_id]['explanation_plot']
|
| 175 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
def parse_inputs(age, gender, CNA_events, mutations):
|
| 177 |
# Normalization of age input
|
| 178 |
age = age
|
|
|
|
| 181 |
except:
|
| 182 |
raise gr.Error('Age should be a positive numerical value')
|
| 183 |
combined_cohort_age_stats = None
|
| 184 |
+
combined_cohort_age_stats_path = './data/combined_cohort_age_stats.pkl'
|
| 185 |
with open(combined_cohort_age_stats_path, "rb") as fp:
|
| 186 |
combined_cohort_age_stats = pickle.load(fp)
|
| 187 |
age = (age - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean']
|
|
|
|
| 277 |
|
| 278 |
# Parse input features
|
| 279 |
features = parse_inputs(age, gender, CNA_events, mutations)
|
| 280 |
+
all_features = pd.read_csv('./data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
|
|
|
|
| 281 |
columns_to_add = {column: [0] * len(features) for column in all_features if column not in features.columns}
|
| 282 |
print(columns_to_add)
|
| 283 |
for column in features.columns:
|
|
|
|
| 382 |
# Generate explanation plot
|
| 383 |
sample_info = f'Prediction: {target}\nPrediction probability: {probability:.3f}'
|
| 384 |
feature_group_to_features_dict, feature_to_feature_group_dict = utils.partition_feature_names_by_group(features.columns)
|
| 385 |
+
explanation_plot = utils.get_individual_pred_interpretation(shap_pred_sample_df, feature_sample_df, feature_group_to_features_dict, feature_to_feature_group_dict,sample_info=sample_info, filename=f'{sample_id}_{target}_plot', filepath='./others_prediction_explanation', save_plot=True)
|
| 386 |
return explanation_plot
|
| 387 |
|
| 388 |
def show_row(value):
|
codes/process_features.py
CHANGED
|
@@ -17,7 +17,7 @@ from absl import app, flags
|
|
| 17 |
import utils
|
| 18 |
|
| 19 |
FLAGS = flags.FLAGS
|
| 20 |
-
_FILEPATH = '.
|
| 21 |
flags.DEFINE_string('config', None, 'Cancer centers to incorporate for feature processing (genie, profile_dfci, or both)')
|
| 22 |
flags.DEFINE_string('filename_suffix', None, 'Filename suffix for processed files.')
|
| 23 |
|
|
@@ -179,7 +179,7 @@ def main(argv):
|
|
| 179 |
df_cup_samples_dfci = df_samples_dfci.loc[df_samples_dfci.PRIMARY_CANCER_DIAGNOSIS.isin(cup_subtypes)].copy()
|
| 180 |
df_cup_samples_dfci.CANCER_TYPE = 'Cancer of Unknown Primary'
|
| 181 |
# Get OncoTree-based cancer types to include.
|
| 182 |
-
with open('.
|
| 183 |
cancers_to_include_dic = pickle.load(handle)
|
| 184 |
if config in ['genie', 'both']:
|
| 185 |
# Process patient data based on cancer types to include.
|
|
@@ -218,7 +218,7 @@ def main(argv):
|
|
| 218 |
cup_samples_ids=df_cup_samples_dfci.UNIQUE_SAMPLE_ID.values)
|
| 219 |
print('Exporting features and labels of interest...')
|
| 220 |
# Load features onconpc pickle file
|
| 221 |
-
with open('.
|
| 222 |
features_onconpc = pickle.load(fp)
|
| 223 |
if config == 'both':
|
| 224 |
# Combine features and labels across centers.
|
|
@@ -228,20 +228,20 @@ def main(argv):
|
|
| 228 |
df_features_cup = df_features_cup[features_onconpc]
|
| 229 |
df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
|
| 230 |
# Export combined (Profile & GENIE) features and labels.
|
| 231 |
-
df_features_combined.to_csv(f'.
|
| 232 |
-
df_labels_combined.to_csv(f'.
|
| 233 |
-
df_features_cup.to_csv(f'.
|
| 234 |
elif config == 'genie':
|
| 235 |
df_features_genie = utils.zero_pad_missing_features(df_features_genie, features_onconpc)
|
| 236 |
df_features_genie.columns = utils.standardize_feat_names(df_features_genie.columns)
|
| 237 |
-
df_features_genie.to_csv(f'.
|
| 238 |
-
df_labels_genie.to_csv(f'.
|
| 239 |
elif config == 'profile_dfci':
|
| 240 |
df_features_dfci.columns = utils.standardize_feat_names(df_features_dfci.columns)
|
| 241 |
df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
|
| 242 |
-
df_features_dfci.to_csv(f'.
|
| 243 |
-
df_labels_dfci.to_csv(f'.
|
| 244 |
-
df_features_cup.to_csv(f'.
|
| 245 |
return
|
| 246 |
|
| 247 |
if __name__ == '__main__':
|
|
|
|
| 17 |
import utils
|
| 18 |
|
| 19 |
FLAGS = flags.FLAGS
|
| 20 |
+
_FILEPATH = './data/seq_panel_data'
|
| 21 |
flags.DEFINE_string('config', None, 'Cancer centers to incorporate for feature processing (genie, profile_dfci, or both)')
|
| 22 |
flags.DEFINE_string('filename_suffix', None, 'Filename suffix for processed files.')
|
| 23 |
|
|
|
|
| 179 |
df_cup_samples_dfci = df_samples_dfci.loc[df_samples_dfci.PRIMARY_CANCER_DIAGNOSIS.isin(cup_subtypes)].copy()
|
| 180 |
df_cup_samples_dfci.CANCER_TYPE = 'Cancer of Unknown Primary'
|
| 181 |
# Get OncoTree-based cancer types to include.
|
| 182 |
+
with open('./data/cancer_type_to_oncotree_subtypes_dict.pkl', 'rb') as handle:
|
| 183 |
cancers_to_include_dic = pickle.load(handle)
|
| 184 |
if config in ['genie', 'both']:
|
| 185 |
# Process patient data based on cancer types to include.
|
|
|
|
| 218 |
cup_samples_ids=df_cup_samples_dfci.UNIQUE_SAMPLE_ID.values)
|
| 219 |
print('Exporting features and labels of interest...')
|
| 220 |
# Load features onconpc pickle file
|
| 221 |
+
with open('./data/features_onconpc.pkl', "rb") as fp:
|
| 222 |
features_onconpc = pickle.load(fp)
|
| 223 |
if config == 'both':
|
| 224 |
# Combine features and labels across centers.
|
|
|
|
| 228 |
df_features_cup = df_features_cup[features_onconpc]
|
| 229 |
df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
|
| 230 |
# Export combined (Profile & GENIE) features and labels.
|
| 231 |
+
df_features_combined.to_csv(f'./data/features_combined_{filename_suffix}', sep='\t', index=True)
|
| 232 |
+
df_labels_combined.to_csv(f'./data/labels_combined_{filename_suffix}', sep='\t', index=True)
|
| 233 |
+
df_features_cup.to_csv(f'./data/features_combined_cup_{filename_suffix}', sep='\t', index=True)
|
| 234 |
elif config == 'genie':
|
| 235 |
df_features_genie = utils.zero_pad_missing_features(df_features_genie, features_onconpc)
|
| 236 |
df_features_genie.columns = utils.standardize_feat_names(df_features_genie.columns)
|
| 237 |
+
df_features_genie.to_csv(f'./data/features_genie_{filename_suffix}', sep='\t', index=True)
|
| 238 |
+
df_labels_genie.to_csv(f'./data/labels_genie_{filename_suffix}', sep='\t', index=True)
|
| 239 |
elif config == 'profile_dfci':
|
| 240 |
df_features_dfci.columns = utils.standardize_feat_names(df_features_dfci.columns)
|
| 241 |
df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
|
| 242 |
+
df_features_dfci.to_csv(f'./data/features_dfci_{filename_suffix}', sep='\t', index=True)
|
| 243 |
+
df_labels_dfci.to_csv(f'./data/labels_dfci_{filename_suffix}', sep='\t', index=True)
|
| 244 |
+
df_features_cup.to_csv(f'./data/features_dfci_cup_{filename_suffix}', sep='\t', index=True)
|
| 245 |
return
|
| 246 |
|
| 247 |
if __name__ == '__main__':
|
codes/utils.py
CHANGED
|
@@ -159,8 +159,8 @@ def get_individual_pred_interpretation(shap_pred_sample_df: pd.DataFrame,
|
|
| 159 |
feature_to_feature_group_dict: dict,
|
| 160 |
sample_info: str = None,
|
| 161 |
filename: str = None,
|
| 162 |
-
filepath: str = '.
|
| 163 |
-
cohort_age_stats_path: str = '.
|
| 164 |
top_feature_num: int = 10,
|
| 165 |
save_plot: bool = False):
|
| 166 |
"""
|
|
@@ -278,8 +278,8 @@ def get_onconpc_prediction_explanations(query_ids: List[str],
|
|
| 278 |
shaps: np.array,
|
| 279 |
df_features_genie: pd.DataFrame,
|
| 280 |
cancer_types_to_consider: List[str],
|
| 281 |
-
filepath: str='.
|
| 282 |
-
cohort_age_stats_path: str='.
|
| 283 |
save_plot: bool=False,
|
| 284 |
) -> List[Mapping[str, Any]]:
|
| 285 |
"""
|
|
@@ -456,7 +456,7 @@ def get_snv_in_trinuc_context(df_mutations: pd.DataFrame,
|
|
| 456 |
return df_trinuc_feats
|
| 457 |
|
| 458 |
def obtain_mutation_signatures(df_trinuc_feats: pd.DataFrame,
|
| 459 |
-
weights_filepath: str='.
|
| 460 |
) -> pd.DataFrame:
|
| 461 |
"""Transforms tri-nucleotide features into mutation signature based features.
|
| 462 |
|
|
|
|
| 159 |
feature_to_feature_group_dict: dict,
|
| 160 |
sample_info: str = None,
|
| 161 |
filename: str = None,
|
| 162 |
+
filepath: str = './others_prediction_explanation',
|
| 163 |
+
cohort_age_stats_path: str = './data/combined_cohort_age_stats.pkl',
|
| 164 |
top_feature_num: int = 10,
|
| 165 |
save_plot: bool = False):
|
| 166 |
"""
|
|
|
|
| 278 |
shaps: np.array,
|
| 279 |
df_features_genie: pd.DataFrame,
|
| 280 |
cancer_types_to_consider: List[str],
|
| 281 |
+
filepath: str='./others_prediction_explanation',
|
| 282 |
+
cohort_age_stats_path: str='./data/combined_cohort_age_stats.pkl',
|
| 283 |
save_plot: bool=False,
|
| 284 |
) -> List[Mapping[str, Any]]:
|
| 285 |
"""
|
|
|
|
| 456 |
return df_trinuc_feats
|
| 457 |
|
| 458 |
def obtain_mutation_signatures(df_trinuc_feats: pd.DataFrame,
|
| 459 |
+
weights_filepath: str='./data/mutation_signatures/sigProfiler*.csv'
|
| 460 |
) -> pd.DataFrame:
|
| 461 |
"""Transforms tri-nucleotide features into mutation signature based features.
|
| 462 |
|
codes/utils_training.py
CHANGED
|
@@ -293,7 +293,7 @@ def perform_k_fold(data: pd.DataFrame,
|
|
| 293 |
if save_model_name is not None:
|
| 294 |
if not os.path.exists('../models'):
|
| 295 |
os.makedirs('../models/models')
|
| 296 |
-
xg_clf.save_model(f'.
|
| 297 |
|
| 298 |
# Get prediction probabilities on val data.
|
| 299 |
pred_probs_on_val_df = pd.DataFrame(pred_probs_on_val, columns=cancer_types, index=X_val.index)
|
|
|
|
| 293 |
if save_model_name is not None:
|
| 294 |
if not os.path.exists('../models'):
|
| 295 |
os.makedirs('../models/models')
|
| 296 |
+
xg_clf.save_model(f'./models/xgboost_{save_model_name}_{k}.json')
|
| 297 |
|
| 298 |
# Get prediction probabilities on val data.
|
| 299 |
pred_probs_on_val_df = pd.DataFrame(pred_probs_on_val, columns=cancer_types, index=X_val.index)
|