jenniferzhou33 commited on
Commit
30e4166
·
1 Parent(s): 0f4d482

file paths

Browse files
codes/gradio_utils.py CHANGED
@@ -49,7 +49,7 @@ cancer_types_to_consider = ['Acute Myeloid Leukemia', 'Bladder Urothelial Carcin
49
  'Pancreatic Neuroendocrine Tumor', 'Pleural Mesothelioma', 'Prostate Adenocarcinoma', 'Renal Cell Carcinoma',
50
  'Well-Differentiated Thyroid Cancer']
51
 
52
- all_features = pd.read_csv('../data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
53
 
54
  def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
55
 
@@ -140,17 +140,17 @@ def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
140
  samples_df,
141
  mutations_df,
142
  cna_df,
143
- features_onconpc_path='../data/features_onconpc.pkl',
144
- combined_cohort_age_stats_path='../data/combined_cohort_age_stats.pkl',
145
- mut_sig_weights_filepath='../data/mutation_signatures/sigProfiler*.csv'
146
  )
147
- df_features_genie_final.to_csv('../data/onconpc_features2.csv', index=True)
148
  sample_id = tumor_id
149
  features = df_features_genie_final
150
 
151
  # load fully trained OncoNPC model
152
  xgb_onconpc = xgb.Booster()
153
- xgb_onconpc.load_model('../model/xgboost_v1.7.6_OncoNPC_full.json')
154
 
155
  # predict primary sites of CUP tumors
156
  predictions = utils.get_xgboost_latest_cancer_type_preds(xgb_onconpc,
@@ -173,67 +173,6 @@ def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
173
 
174
  return get_top3(predictions, tumor_id), gr.Markdown(markdown_text), results[tumor_id]['explanation_plot']
175
 
176
- # def parse_inputs(age, gender, CNA_events, mutations, sample_id):
177
-
178
- # # Normalization of age input
179
- # combined_cohort_age_stats = None
180
- # combined_cohort_age_stats_path = '../data/combined_cohort_age_stats.pkl'
181
- # with open(combined_cohort_age_stats_path, "rb") as fp:
182
- # combined_cohort_age_stats = pickle.load(fp)
183
- # age = (age - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean']
184
-
185
- # # Initialize data dictionary and populate with age, sex, and sample id
186
- # data = {'AGE_AT_SEQ_REPORT': age, 'SEX':gender, 'SAMPLE_ID':sample_id}
187
- # df_patients = pd.DataFrame([data])
188
-
189
- # # Process CNA events: Expected format [[CNA, val], [CNA, val]]
190
- # if len(CNA_events) > 0:
191
- # CNA_events = CNA_events.split('|')
192
- # for i in range(len(CNA_events)):
193
- # # Split each event into CNA and value, and cast the value to integer
194
- # CNA, val = CNA_events[i].split()
195
- # CNA_events[i] = [CNA + ' CNA', int(val)] # Cast val to integer
196
- # else:
197
- # CNA_events = []
198
-
199
- # CNA_dict = {}
200
- # # Add CNA events to data dictionary
201
- # for CNA, val in CNA_events:
202
- # CNA_dict[CNA] = val
203
-
204
- # df_cna = pd.DataFrame([CNA_dict])
205
-
206
- # # Process mutations: Expected format [mut1, mut2, etc.]
207
- # if len(mutations) > 0:
208
- # mutations = mutations.split('| ')
209
- # for i in range(len(mutations)):
210
- # # Split each mutation entry and strip white space
211
- # mutations[i] = ['manual input'] + mutations[i].split(', ')
212
- # mutations[i] = [m.strip() for m in mutations[i]] # Strip white space
213
- # else:
214
- # mutations = []
215
-
216
- # # Define mutation columns for DataFrame and create mutation DataFrame
217
- # mutation_columns = ["UNIQUE_SAMPLE_ID", "HUGO_SYMBOL", "CHROMOSOME", "POSITION", "REF_ALLELE", "ALT_ALLELE"]
218
- # mutation_full_df = pd.DataFrame(mutations, columns=mutation_columns) if mutations else pd.DataFrame(columns=mutation_columns)
219
- # mutation_partial_df = mutation_full_df[['UNIQUE_SAMPLE_ID', 'HUGO_SYMBOL']]
220
- # mutation_partial_df = mutation_partial_df.rename(columns={'UNIQUE_SAMPLE_ID':'Tumor_Sample_Barcode', 'HUGO_SYMBOL':'Hugo_Symbol'})
221
- # mutation_df = mutation_full_df.drop('HUGO_SYMBOL', axis=1, inplace=False)
222
-
223
- # # Save mutation data to a CSV file
224
- # mutation_df.to_csv('./mutation_input.csv', index=False)
225
-
226
- # # Get base substitution file and read into DataFrame
227
- # base_sub_file = deconstructSigs.get_base_substitutions()
228
- # df_trinuc_feats = pd.read_csv(base_sub_file)
229
- # # Obtain mutation signatures
230
- # mutation_signatures = utils.obtain_mutation_signatures(df_trinuc_feats)
231
-
232
-
233
- # # Return processed features
234
- # return utils.pre_process_features_genie(mutation_partial_df, df_cna, mutation_signatures, df_patients)[0] # just features, not labels
235
-
236
-
237
  def parse_inputs(age, gender, CNA_events, mutations):
238
  # Normalization of age input
239
  age = age
@@ -242,7 +181,7 @@ def parse_inputs(age, gender, CNA_events, mutations):
242
  except:
243
  raise gr.Error('Age should be a positive numerical value')
244
  combined_cohort_age_stats = None
245
- combined_cohort_age_stats_path = '../data/combined_cohort_age_stats.pkl'
246
  with open(combined_cohort_age_stats_path, "rb") as fp:
247
  combined_cohort_age_stats = pickle.load(fp)
248
  age = (age - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean']
@@ -338,8 +277,7 @@ def get_preds_min_info(age, gender, CNA_events, mutations, output='Top Predictio
338
 
339
  # Parse input features
340
  features = parse_inputs(age, gender, CNA_events, mutations)
341
- features.to_csv('../genie1.csv', index=False)
342
- all_features = pd.read_csv('../data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
343
  columns_to_add = {column: [0] * len(features) for column in all_features if column not in features.columns}
344
  print(columns_to_add)
345
  for column in features.columns:
@@ -444,7 +382,7 @@ def update_image(target):
444
  # Generate explanation plot
445
  sample_info = f'Prediction: {target}\nPrediction probability: {probability:.3f}'
446
  feature_group_to_features_dict, feature_to_feature_group_dict = utils.partition_feature_names_by_group(features.columns)
447
- explanation_plot = utils.get_individual_pred_interpretation(shap_pred_sample_df, feature_sample_df, feature_group_to_features_dict, feature_to_feature_group_dict,sample_info=sample_info, filename=f'{sample_id}_{target}_plot', filepath='../others_prediction_explanation', save_plot=True)
448
  return explanation_plot
449
 
450
  def show_row(value):
 
49
  'Pancreatic Neuroendocrine Tumor', 'Pleural Mesothelioma', 'Prostate Adenocarcinoma', 'Renal Cell Carcinoma',
50
  'Well-Differentiated Thyroid Cancer']
51
 
52
+ all_features = pd.read_csv('./data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
53
 
54
  def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
55
 
 
140
  samples_df,
141
  mutations_df,
142
  cna_df,
143
+ features_onconpc_path='./data/features_onconpc.pkl',
144
+ combined_cohort_age_stats_path='./data/combined_cohort_age_stats.pkl',
145
+ mut_sig_weights_filepath='./data/mutation_signatures/sigProfiler*.csv'
146
  )
147
+ df_features_genie_final.to_csv('./data/onconpc_features2.csv', index=True)
148
  sample_id = tumor_id
149
  features = df_features_genie_final
150
 
151
  # load fully trained OncoNPC model
152
  xgb_onconpc = xgb.Booster()
153
+ xgb_onconpc.load_model('./model/xgboost_v1.7.6_OncoNPC_full.json')
154
 
155
  # predict primary sites of CUP tumors
156
  predictions = utils.get_xgboost_latest_cancer_type_preds(xgb_onconpc,
 
173
 
174
  return get_top3(predictions, tumor_id), gr.Markdown(markdown_text), results[tumor_id]['explanation_plot']
175
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  def parse_inputs(age, gender, CNA_events, mutations):
177
  # Normalization of age input
178
  age = age
 
181
  except:
182
  raise gr.Error('Age should be a positive numerical value')
183
  combined_cohort_age_stats = None
184
+ combined_cohort_age_stats_path = './data/combined_cohort_age_stats.pkl'
185
  with open(combined_cohort_age_stats_path, "rb") as fp:
186
  combined_cohort_age_stats = pickle.load(fp)
187
  age = (age - combined_cohort_age_stats['Age_mean']) / combined_cohort_age_stats['Std_mean']
 
277
 
278
  # Parse input features
279
  features = parse_inputs(age, gender, CNA_events, mutations)
280
+ all_features = pd.read_csv('./data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
 
281
  columns_to_add = {column: [0] * len(features) for column in all_features if column not in features.columns}
282
  print(columns_to_add)
283
  for column in features.columns:
 
382
  # Generate explanation plot
383
  sample_info = f'Prediction: {target}\nPrediction probability: {probability:.3f}'
384
  feature_group_to_features_dict, feature_to_feature_group_dict = utils.partition_feature_names_by_group(features.columns)
385
+ explanation_plot = utils.get_individual_pred_interpretation(shap_pred_sample_df, feature_sample_df, feature_group_to_features_dict, feature_to_feature_group_dict,sample_info=sample_info, filename=f'{sample_id}_{target}_plot', filepath='./others_prediction_explanation', save_plot=True)
386
  return explanation_plot
387
 
388
  def show_row(value):
codes/process_features.py CHANGED
@@ -17,7 +17,7 @@ from absl import app, flags
17
  import utils
18
 
19
  FLAGS = flags.FLAGS
20
- _FILEPATH = '../data/seq_panel_data'
21
  flags.DEFINE_string('config', None, 'Cancer centers to incorporate for feature processing (genie, profile_dfci, or both)')
22
  flags.DEFINE_string('filename_suffix', None, 'Filename suffix for processed files.')
23
 
@@ -179,7 +179,7 @@ def main(argv):
179
  df_cup_samples_dfci = df_samples_dfci.loc[df_samples_dfci.PRIMARY_CANCER_DIAGNOSIS.isin(cup_subtypes)].copy()
180
  df_cup_samples_dfci.CANCER_TYPE = 'Cancer of Unknown Primary'
181
  # Get OncoTree-based cancer types to include.
182
- with open('../data/cancer_type_to_oncotree_subtypes_dict.pkl', 'rb') as handle:
183
  cancers_to_include_dic = pickle.load(handle)
184
  if config in ['genie', 'both']:
185
  # Process patient data based on cancer types to include.
@@ -218,7 +218,7 @@ def main(argv):
218
  cup_samples_ids=df_cup_samples_dfci.UNIQUE_SAMPLE_ID.values)
219
  print('Exporting features and labels of interest...')
220
  # Load features onconpc pickle file
221
- with open('../data/features_onconpc.pkl', "rb") as fp:
222
  features_onconpc = pickle.load(fp)
223
  if config == 'both':
224
  # Combine features and labels across centers.
@@ -228,20 +228,20 @@ def main(argv):
228
  df_features_cup = df_features_cup[features_onconpc]
229
  df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
230
  # Export combined (Profile & GENIE) features and labels.
231
- df_features_combined.to_csv(f'../data/features_combined_{filename_suffix}', sep='\t', index=True)
232
- df_labels_combined.to_csv(f'../data/labels_combined_{filename_suffix}', sep='\t', index=True)
233
- df_features_cup.to_csv(f'../data/features_combined_cup_{filename_suffix}', sep='\t', index=True)
234
  elif config == 'genie':
235
  df_features_genie = utils.zero_pad_missing_features(df_features_genie, features_onconpc)
236
  df_features_genie.columns = utils.standardize_feat_names(df_features_genie.columns)
237
- df_features_genie.to_csv(f'../data/features_genie_{filename_suffix}', sep='\t', index=True)
238
- df_labels_genie.to_csv(f'../data/labels_genie_{filename_suffix}', sep='\t', index=True)
239
  elif config == 'profile_dfci':
240
  df_features_dfci.columns = utils.standardize_feat_names(df_features_dfci.columns)
241
  df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
242
- df_features_dfci.to_csv(f'../data/features_dfci_{filename_suffix}', sep='\t', index=True)
243
- df_labels_dfci.to_csv(f'../data/labels_dfci_{filename_suffix}', sep='\t', index=True)
244
- df_features_cup.to_csv(f'../data/features_dfci_cup_{filename_suffix}', sep='\t', index=True)
245
  return
246
 
247
  if __name__ == '__main__':
 
17
  import utils
18
 
19
  FLAGS = flags.FLAGS
20
+ _FILEPATH = './data/seq_panel_data'
21
  flags.DEFINE_string('config', None, 'Cancer centers to incorporate for feature processing (genie, profile_dfci, or both)')
22
  flags.DEFINE_string('filename_suffix', None, 'Filename suffix for processed files.')
23
 
 
179
  df_cup_samples_dfci = df_samples_dfci.loc[df_samples_dfci.PRIMARY_CANCER_DIAGNOSIS.isin(cup_subtypes)].copy()
180
  df_cup_samples_dfci.CANCER_TYPE = 'Cancer of Unknown Primary'
181
  # Get OncoTree-based cancer types to include.
182
+ with open('./data/cancer_type_to_oncotree_subtypes_dict.pkl', 'rb') as handle:
183
  cancers_to_include_dic = pickle.load(handle)
184
  if config in ['genie', 'both']:
185
  # Process patient data based on cancer types to include.
 
218
  cup_samples_ids=df_cup_samples_dfci.UNIQUE_SAMPLE_ID.values)
219
  print('Exporting features and labels of interest...')
220
  # Load features onconpc pickle file
221
+ with open('./data/features_onconpc.pkl', "rb") as fp:
222
  features_onconpc = pickle.load(fp)
223
  if config == 'both':
224
  # Combine features and labels across centers.
 
228
  df_features_cup = df_features_cup[features_onconpc]
229
  df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
230
  # Export combined (Profile & GENIE) features and labels.
231
+ df_features_combined.to_csv(f'./data/features_combined_{filename_suffix}', sep='\t', index=True)
232
+ df_labels_combined.to_csv(f'./data/labels_combined_{filename_suffix}', sep='\t', index=True)
233
+ df_features_cup.to_csv(f'./data/features_combined_cup_{filename_suffix}', sep='\t', index=True)
234
  elif config == 'genie':
235
  df_features_genie = utils.zero_pad_missing_features(df_features_genie, features_onconpc)
236
  df_features_genie.columns = utils.standardize_feat_names(df_features_genie.columns)
237
+ df_features_genie.to_csv(f'./data/features_genie_{filename_suffix}', sep='\t', index=True)
238
+ df_labels_genie.to_csv(f'./data/labels_genie_{filename_suffix}', sep='\t', index=True)
239
  elif config == 'profile_dfci':
240
  df_features_dfci.columns = utils.standardize_feat_names(df_features_dfci.columns)
241
  df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)
242
+ df_features_dfci.to_csv(f'./data/features_dfci_{filename_suffix}', sep='\t', index=True)
243
+ df_labels_dfci.to_csv(f'./data/labels_dfci_{filename_suffix}', sep='\t', index=True)
244
+ df_features_cup.to_csv(f'./data/features_dfci_cup_{filename_suffix}', sep='\t', index=True)
245
  return
246
 
247
  if __name__ == '__main__':
codes/utils.py CHANGED
@@ -159,8 +159,8 @@ def get_individual_pred_interpretation(shap_pred_sample_df: pd.DataFrame,
159
  feature_to_feature_group_dict: dict,
160
  sample_info: str = None,
161
  filename: str = None,
162
- filepath: str = '../others_prediction_explanation',
163
- cohort_age_stats_path: str = '../data/combined_cohort_age_stats.pkl',
164
  top_feature_num: int = 10,
165
  save_plot: bool = False):
166
  """
@@ -278,8 +278,8 @@ def get_onconpc_prediction_explanations(query_ids: List[str],
278
  shaps: np.array,
279
  df_features_genie: pd.DataFrame,
280
  cancer_types_to_consider: List[str],
281
- filepath: str='../others_prediction_explanation',
282
- cohort_age_stats_path: str='../data/combined_cohort_age_stats.pkl',
283
  save_plot: bool=False,
284
  ) -> List[Mapping[str, Any]]:
285
  """
@@ -456,7 +456,7 @@ def get_snv_in_trinuc_context(df_mutations: pd.DataFrame,
456
  return df_trinuc_feats
457
 
458
  def obtain_mutation_signatures(df_trinuc_feats: pd.DataFrame,
459
- weights_filepath: str='../data/mutation_signatures/sigProfiler*.csv'
460
  ) -> pd.DataFrame:
461
  """Transforms tri-nucleotide features into mutation signature based features.
462
 
 
159
  feature_to_feature_group_dict: dict,
160
  sample_info: str = None,
161
  filename: str = None,
162
+ filepath: str = './others_prediction_explanation',
163
+ cohort_age_stats_path: str = './data/combined_cohort_age_stats.pkl',
164
  top_feature_num: int = 10,
165
  save_plot: bool = False):
166
  """
 
278
  shaps: np.array,
279
  df_features_genie: pd.DataFrame,
280
  cancer_types_to_consider: List[str],
281
+ filepath: str='./others_prediction_explanation',
282
+ cohort_age_stats_path: str='./data/combined_cohort_age_stats.pkl',
283
  save_plot: bool=False,
284
  ) -> List[Mapping[str, Any]]:
285
  """
 
456
  return df_trinuc_feats
457
 
458
  def obtain_mutation_signatures(df_trinuc_feats: pd.DataFrame,
459
+ weights_filepath: str='./data/mutation_signatures/sigProfiler*.csv'
460
  ) -> pd.DataFrame:
461
  """Transforms tri-nucleotide features into mutation signature based features.
462
 
codes/utils_training.py CHANGED
@@ -293,7 +293,7 @@ def perform_k_fold(data: pd.DataFrame,
293
  if save_model_name is not None:
294
  if not os.path.exists('../models'):
295
  os.makedirs('../models/models')
296
- xg_clf.save_model(f'../models/xgboost_{save_model_name}_{k}.json')
297
 
298
  # Get prediction probabilities on val data.
299
  pred_probs_on_val_df = pd.DataFrame(pred_probs_on_val, columns=cancer_types, index=X_val.index)
 
293
  if save_model_name is not None:
294
  if not os.path.exists('../models'):
295
  os.makedirs('../models/models')
296
+ xg_clf.save_model(f'./models/xgboost_{save_model_name}_{k}.json')
297
 
298
  # Get prediction probabilities on val data.
299
  pred_probs_on_val_df = pd.DataFrame(pred_probs_on_val, columns=cancer_types, index=X_val.index)