itmoon7 commited on
Commit
5484f77
·
verified ·
1 Parent(s): 6973f9d

Fix Gradio cache cleanup breaking update_image

Browse files
Files changed (1) hide show
  1. codes/gradio_utils.py +528 -253
codes/gradio_utils.py CHANGED
@@ -10,20 +10,54 @@ import pickle
10
  import pandas as pd
11
  import xgboost as xgb
12
 
13
- global image # path to explanation plot, defined as global for the purposes of update
14
- global features
15
-
16
- def cleanup(uploaded_files=None):
17
- """Clear global state and temp files after each request."""
18
- global features, predictions, sample_id
19
- features = None
20
- predictions = None
21
- sample_id = None
22
- if uploaded_files:
23
- for f in uploaded_files:
24
- if f is not None and hasattr(f, 'name') and os.path.exists(f.name):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  os.remove(f.name)
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  markdown_text = """
28
  **Feature Category Explanations**
29
 
@@ -62,128 +96,200 @@ cancer_types_to_consider = ['Acute Myeloid Leukemia', 'Bladder Urothelial Carcin
62
 
63
  all_features = pd.read_csv('./data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
64
 
65
- def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id):
66
-
67
  """
68
- Generates predictions and explanations for given tumor samples using OncoNPC model.
69
-
70
- This function processes patient, sample, mutation, and CNA data to predict primary sites of
71
- Cancer of Unknown Primary (CUP) tumors. It also provides a bar chart of SHAP values to explain the predictions.
72
 
73
  Args:
74
- patients_file: A csv file object representing patient data.
75
- samples_file: A csv file object representing sample data.
76
- mutations_file: A csv file object representing mutation data.
77
- cna_file: A csv file object representing CNA (Copy Number Alterations) data.
78
- tumor_id: The ID of the tumor.
 
 
 
 
 
 
79
 
80
  Returns:
81
- A tuple containing:
82
- A string containing the top 3 most probable cancers along with their predicted probabilities.
83
- The filepath to the SHAP value bar chart explaining the prediction for the given tumor ID.
 
 
84
  """
85
- # make sure all files have been uploaded
86
- if patients_file is None or samples_file is None or cna_file is None or mutations_file is None:
87
- raise gr.Error('Please make sure all files have been properly uploaded')
88
-
89
- # convert files to data frames
90
- if patients_file.name[-4:] == '.txt':
91
- patients_df = pd.read_csv(patients_file.name, sep='\t')
92
- elif patients_file.name[-4:] == '.csv':
93
- patients_df = pd.read_csv(patients_file.name)
 
 
94
  else:
95
- raise gr.Error('Please make sure all files are .txt or .csv files')
96
-
97
- if samples_file.name[-4:] == '.txt':
98
- samples_df = pd.read_csv(samples_file.name, sep='\t')
99
- elif patients_file.name[-4:] == '.csv':
100
- samples_df = pd.read_csv(samples_file.name)
101
- else:
102
- raise gr.Error('Please make sure all files are .txt or .csv files')
103
-
104
- if mutations_file.name[-4:] == '.txt':
105
- mutations_df = pd.read_csv(mutations_file.name, sep='\t')
106
- elif mutations_file.name[-4:] == '.csv':
107
- mutations_df = pd.read_csv(mutations_file.name)
108
- else:
109
- raise gr.Error('Please make sure all files are .txt or .csv files')
110
-
111
- if cna_file.name[-4:] == '.txt':
112
- cna_df = pd.read_csv(cna_file.name, sep='\t')
113
- elif cna_file.name[-4:] == '.csv':
114
- cna_df = pd.read_csv(cna_file.name)
115
- else:
116
- raise gr.Error('Please make sure all files are .txt or .csv files')
117
-
118
- if tumor_id is None:
119
- raise gr.Error('Tumor Sample ID cannot be empty')
120
- else:
121
- print(tumor_id)
122
-
123
- patients_columns = set(['PATIENT_ID', 'SEX'])
124
- for col in patients_columns:
125
- if col not in set(patients_df.columns.to_list()):
126
- raise gr.Error(f'Patients File: Expected columns: ' + str(list(patients_columns)) + ', Received: ' + str(patients_df.columns.to_list()))
127
-
128
- samples_columns = set(['PATIENT_ID', 'AGE_AT_SEQ_REPORT', 'SAMPLE_ID'])
129
- for col in samples_columns:
130
- if col not in set(samples_df.columns.to_list()):
131
- raise gr.Error(f'Samples File: Expected columns: ' + str(list(samples_columns)) + ', Received: ' + str(samples_df.columns.to_list()))
132
-
133
- mutations_columns = set(['Tumor_Sample_Barcode', 'Hugo_Symbol', 'Chromosome', 'Start_Position', 'Reference_Allele', 'Tumor_Seq_Allele2'])
134
- for col in mutations_columns:
135
- if col not in set(mutations_df.columns.to_list()):
136
- raise gr.Error(f'Mutations File: Expected columns: ' + str(list(mutations_columns)) + ', Received: ' + str(mutations_df.columns.to_list()))
137
-
138
- cna_columns = set(['Hugo_Symbol'])
139
- for col in cna_columns:
140
- if col not in set(cna_df.columns.to_list()):
141
- raise gr.Error(f'CNA File: Expected columns: ' + str(list(cna_columns)) + ', Received: ' + str(cna_df.columns.to_list()))
142
-
143
- # declared as global variables to generate plots in update_image function
144
- global sample_id
145
- global features
146
- global predictions
147
-
148
- # get features and labels for OncoNPC predictive inference
149
- df_features_genie_final, df_labels_genie = utils.get_onconpc_features_from_raw_data(
150
- patients_df,
151
- samples_df,
152
- mutations_df,
153
- cna_df,
154
- features_onconpc_path='./data/features_onconpc.pkl',
155
- combined_cohort_age_stats_path='./data/combined_cohort_age_stats.pkl',
156
- mut_sig_weights_filepath='./data/mutation_signatures/sigProfiler*.csv'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  )
158
- sample_id = tumor_id
159
- features = df_features_genie_final
160
-
161
- # load fully trained OncoNPC model
162
- xgb_onconpc = xgb.Booster()
163
- xgb_onconpc.load_model('./model/xgboost_v1.7.6_OncoNPC_full.json')
164
-
165
- # predict primary sites of CUP tumors
166
- predictions = utils.get_xgboost_latest_cancer_type_preds(xgb_onconpc,
167
- features,
168
- cancer_types_to_consider)
169
-
170
- # get SHAP values for CUP tumors
171
- warnings.filterwarnings('ignore')
172
- shaps = utils.obtain_shap_values_with_latest_xgboost(xgb_onconpc, df_features_genie_final)
173
-
174
 
175
- query_ids = list(samples_df.SAMPLE_ID.values)
176
-
177
- # results is structured such that:
178
- # results_dict[query_id] = {'pred_prob': pred_prob,'pred_cancer': pred_cancer,'explanation_plot': full_filename}
179
- results = utils.get_onconpc_prediction_explanations(query_ids, predictions, shaps,
180
- df_features_genie_final,
181
- cancer_types_to_consider,
182
- save_plot=True)
183
-
184
- result = get_top3(predictions, tumor_id), gr.Markdown(markdown_text), results[tumor_id]['explanation_plot']
185
- cleanup([patients_file, samples_file, mutations_file, cna_file])
186
- return result
187
 
188
  def parse_inputs(age, gender, CNA_events, mutations):
189
  # Normalization of age input
@@ -272,55 +378,91 @@ def parse_inputs(age, gender, CNA_events, mutations):
272
  # Return the data as a DataFrame
273
  return pd.DataFrame([data])
274
 
275
- def get_preds_min_info(age, gender, CNA_events, mutations, output='Top Prediction'):
276
  """
277
- Generate predictions and explanations for cancer type based on input features.
 
 
 
278
 
279
- Parameters:
280
- age (int or float): The age of the individual.
281
- gender (str): The gender of the individual, either 'male' or 'female'.
282
- CNA_events (str): A string of CNA events, formatted appropriately.
283
- mutations (str): A string of mutation data, formatted appropriately.
284
- output (str, optional): Specifies the type of output; default is 'Top Prediction'.
 
285
 
286
  Returns:
287
- tuple: A tuple containing top 3 predictions and the path to the explanation plot.
 
 
 
 
288
  """
289
- global sample_id
290
- global features
291
- global predictions
292
-
293
- # Parse input features
294
  features = parse_inputs(age, gender, CNA_events, mutations)
295
- all_features = pd.read_csv('./data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
296
- columns_to_add = {column: [0] * len(features) for column in all_features if column not in features.columns}
297
- print(columns_to_add)
298
- for column in features.columns:
299
- if column not in all_features:
300
- features = features.drop(column, axis=1)
301
- new_columns_df = pd.DataFrame(columns_to_add)
302
- features = pd.concat([features, new_columns_df], axis=1)
303
- features = features[all_features]
304
-
305
- # Generate predictions using the XGBoost model
306
- predictions = pd.DataFrame(utils.get_xgboost_latest_cancer_type_preds(xgb_onconpc, features, cancer_types_to_consider))
307
-
308
- # Compute SHAP values for model explanation
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
309
  shaps = utils.obtain_shap_values_with_latest_xgboost(xgb_onconpc, features)
310
 
311
- # Assuming a single sample is being processed
312
  query_ids = [0]
313
- sample_id = 0
314
-
315
- # Generate explanations for the predictions
316
- results = utils.get_onconpc_prediction_explanations(query_ids, predictions, shaps,
317
- features, cancer_types_to_consider,
318
- save_plot=True)
 
 
319
 
320
- # Return the top 3 predictions and the path to the explanation plot
321
- result = get_top3(predictions, 0), gr.Markdown(markdown_text, elem_classes="markdown-box"), results[0]['explanation_plot']
322
- cleanup()
323
- return result
 
 
 
 
 
 
 
 
 
 
324
 
325
  def get_top3(predictions, tumor_sample_id):
326
  """
@@ -378,27 +520,77 @@ def extract_sample_ids(samples_file):
378
  sample_ids = df['SAMPLE_ID'].unique().tolist()
379
  return gr.Dropdown(choices=sample_ids)
380
 
381
- def update_image(target):
382
- global image
383
- global features
384
- global predictions
385
-
386
- shaps_cup = utils.obtain_shap_values_with_latest_xgboost(xgb_onconpc, features) # get shap values
387
 
388
- target_idx = cancer_types_to_consider.index(target) # index of cancer type prediction
389
-
390
- # Get SHAP-based explanation for the prediction
391
- feature_sample_df = features.loc[sample_id] # find the exact tumor sample we're predicting for
392
- shap_pred_cancer_df = pd.DataFrame(shaps_cup[target_idx],
393
- index=features.index,
394
- columns=features.columns)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
395
  shap_pred_sample_df = shap_pred_cancer_df.loc[sample_id]
 
 
 
396
  probability = predictions.loc[sample_id][target]
397
-
398
- # Generate explanation plot
399
- sample_info = f'Prediction: {target}\nPrediction probability: {probability:.3f}'
400
- feature_group_to_features_dict, feature_to_feature_group_dict = utils.partition_feature_names_by_group(features.columns)
401
- explanation_plot = utils.get_individual_pred_interpretation(shap_pred_sample_df, feature_sample_df, feature_group_to_features_dict, feature_to_feature_group_dict,sample_info=sample_info, filename=f'{sample_id}_{target}_plot', filepath='./others_prediction_explanation', save_plot=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
402
  return explanation_plot
403
 
404
  def show_row(value):
@@ -432,102 +624,185 @@ def show_row(value):
432
 
433
  def launch_gradio(server_name, server_port):
434
  """
435
- Launches a Gradio application for cancer type prediction based on various clinical and genomic data.
 
436
 
437
  Args:
438
- server_name (str): Name of the server where the app will be hosted.
439
- server_port (int): Port number for the server.
440
 
441
- This function sets up a Gradio interface with options to input either manual data or CSV files.
442
- The application supports prediction for a predefined list of cancer types based on the input data.
443
  """
444
- cancer_types_to_consider = ['Acute Myeloid Leukemia', 'Bladder Urothelial Carcinoma', 'Cholangiocarcinoma',
445
- 'Colorectal Adenocarcinoma', 'Diffuse Glioma', 'Endometrial Carcinoma',
446
- 'Esophagogastric Adenocarcinoma', 'Gastrointestinal Neuroendocrine Tumors', 'Gastrointestinal Stromal Tumor',
447
- 'Head and Neck Squamous Cell Carcinoma',
448
- 'Invasive Breast Carcinoma', 'Melanoma', 'Meningothelial Tumor',
449
- 'Non-Hodgkin Lymphoma', 'Non-Small Cell Lung Cancer', 'Ovarian Epithelial Tumor', 'Pancreatic Adenocarcinoma',
450
- 'Pancreatic Neuroendocrine Tumor', 'Pleural Mesothelioma', 'Prostate Adenocarcinoma', 'Renal Cell Carcinoma',
451
- 'Well-Differentiated Thyroid Cancer']
452
-
453
- markdown_text = """
454
- **Prediction Explanations**
455
-
456
- **Somatic mutations**: Single nucleotide variants (SNV) and indels. For each gene, the total count of a somatic mutation is encoded as a positive integer feature.
457
-
458
- **Copy number alterations (CNA)**: CNA event for each gene was encoded as a categorical variable with the following five levels:
459
- - `-2`: Deep loss
460
- - `-1`: Single-copy loss
461
- - `0`: No event
462
- - `1`: Low-level gain
463
- - `2`: High-level amplification.
464
-
465
- **Mutation signatures (single-base substitution (SBS))**: Mutation signatures were inferred by calculating the dot product of COSMIC-derived weights and 96 single-base substitutions in a trinucleotide context.
466
- - SBS4: Tobacco smoking
467
- - SBS7: Ultraviolet light exposure
468
- - SBS22: Aristolochic acid exposure
469
- - SBS24: Aflatoxin exposure
470
- - See more details in [COSMIC](https://cancer.sanger.ac.uk/signatures/sbs/).
471
-
472
- **Age/Sex**: Patient age at the time of sequencing and biological sex.
473
-
474
- **SHAP Values**: impact on OncoNPC prediction
475
- - larger magnitude indicates greater impact
476
- """
477
- manual_input_description = 'In this option, you will manually enter patient and tumor data including age at sequencing, sex, copy number alterations (CNA) events, and somatic mutations.'
478
- csv_description = 'In this option, you can directly upload relevant CSV files, including clinical patient, clinical sample, mutations, and CNA data from [cBioPortal](https://www.cbioportal.org) or [AACR GENIE](https://www.aacr.org/professionals/research/aacr-project-genie/). Please see the example data in this [link](https://github.com/itmoon7/onconpc/tree/main/data/mock_n_3_data) <br> Note that column names `Tumor_Sample_Barcode` and `SAMPLE_ID` both refer to the sample identifier column and should be the same.'
479
-
480
- with gr.Blocks(css=".markdown-box { border: 1px solid #E0E0E0; border-radius: 8px; box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05); padding: 15px; margin-bottom: 20px; }") as demo:
481
-
482
- d = gr.Dropdown(label='Select your preferred input method for tumor data to visualize predictions made by OncoNPC', choices=["Manual Inputs", "CSV File"])
483
 
 
484
  with gr.Row(visible=False) as r3:
485
- gr.Markdown(csv_description,elem_classes="markdown-box")
486
  with gr.Row(visible=False) as r4:
487
- gr.Markdown(manual_input_description,elem_classes="markdown-box")
488
 
 
 
 
489
  with gr.Row(visible=False) as r1:
 
 
 
 
490
  with gr.Column():
491
- gr.Markdown("**Clinical Patient Data File:** Upload a file containing patient data. Required columns: `PATIENT_ID`, `SEX`.")
492
  patients_file = gr.File(label="Upload clinical patients data")
493
- gr.Markdown("**Clinical Sample Data File:** Upload a file containing sample data. Required columns: `PATIENT_ID`, `AGE_AT_SEQ_REPORT`, `SAMPLE_ID`.")
 
494
  samples_file = gr.File(label="Upload clinical samples data")
495
- gr.Markdown("**Mutations Data File:** Upload a file containing mutation data. Required columns: `Tumor_Sample_Barcode`, `Hugo_Symbol`, `Chromosome`, `Start_Position`, `Reference_Allele`, `Tumor_Seq_Allele2`.")
 
 
 
 
496
  mutations_file = gr.File(label="Upload mutations data")
497
- gr.Markdown("**CNA Data File:** Upload a file containing CNA data. Required column: `Hugo_Symbol`.")
 
498
  cna_file = gr.File(label="Upload CNA data")
499
- tumor_sample_id = gr.Dropdown(choices=[], label="Tumor Sample ID") # Changed to Dropdown
500
- submit_button = gr.Button("Submit")
 
 
 
501
 
502
  with gr.Column():
503
  predictions_output = gr.Textbox(label="Top 3 Predicted Cancer Types")
504
- category_explanation = gr.Markdown(markdown_text,elem_classes="markdown-box")
505
- image = gr.Image(label="Image Display")
506
- output_selector = gr.Dropdown(choices=cancer_types_to_consider, label="Output Options", filterable=True)
507
- gr.Markdown('For more details regarding OncoNPC feature processing, OncoNPC performance, and clinical implications of OncoNPC predictions, please see our [manuscript](https://www.nature.com/articles/s41591-023-02482-6).', elem_classes="markdown-box")
 
 
 
 
 
 
508
 
 
 
 
 
 
 
 
509
  samples_file.change(extract_sample_ids, inputs=samples_file, outputs=tumor_sample_id)
510
- submit_button.click(get_preds, inputs=[patients_file, samples_file, mutations_file, cna_file, tumor_sample_id], outputs=[predictions_output, category_explanation, image])
511
- output_selector.change(update_image, inputs=output_selector, outputs=[image])
512
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
513
  with gr.Row(visible=False) as r2:
 
 
 
514
  with gr.Column():
515
  age = gr.Number(label="Age")
516
  gender = gr.Radio(choices=["Male", "Female"], label="Sex")
517
- cna_events = gr.Textbox(lines=7, placeholder="Copy Number Alterations\n\nPlease enter the name of each gene followed by its corresponding CNA level. If you are entering data for multiple genes, separate each gene and its CNA level with a vertical bar '|'.\n\nFor example: RAF1 2 | PPARG 2", label="Genes with CNA Events")
518
- mutations = gr.Textbox(lines=7, placeholder="Somatic Mutations\n\nPlease enter the gene name, chromosome number, position of the variant, allele in the reference genome, and alternate allele. If you are entering data for multiple genes, separate each set of information with a vertical bar '|'.\n\nFor example: ERBB2, chr17, 37868208, C, T | CDKN1A, chr6, 36651880, -, GTCAGAACCGGCTGGGGATGTCC", label="Mutations")
519
- submit_button = gr.Button("Submit")
520
- with gr.Column():
521
- predictions_output = gr.Textbox(label="Top 3 Predicted Cancer Types")
522
- category_explanation = gr.Markdown(markdown_text,elem_classes="markdown-box")
523
- image = gr.Image(label="Image Display")
524
- output_selector = gr.Dropdown(choices=cancer_types_to_consider, label="Output Options", filterable=True)
525
- gr.Markdown('For more details regarding OncoNPC feature processing, OncoNPC performance, and clinical implications of OncoNPC predictions, please see our [manuscript](https://www.nature.com/articles/s41591-023-02482-6).', elem_classes="markdown-box")
526
 
527
- submit_button.click(get_preds_min_info, inputs=[age, gender, cna_events, mutations], outputs=[predictions_output, category_explanation, image])
528
- output_selector.change(update_image, inputs=output_selector, outputs=[image])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
529
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
530
  d.change(show_row, d, [r1, r2])
531
  d.change(show_row, d, [r3, r4])
532
-
533
- demo.launch(debug=True, share=False,server_name=server_name, server_port=server_port)
 
 
10
  import pandas as pd
11
  import xgboost as xgb
12
 
13
+
14
+ def cleanup_files(uploaded_files=None):
15
+ """
16
+ Remove uploaded temporary files from disk without touching any session/model state.
17
+
18
+ Args:
19
+ uploaded_files (list | None): Optional list of Gradio `File` objects (or file-like objects)
20
+ that have a `.name` attribute pointing to a local temporary filepath.
21
+
22
+ Returns:
23
+ None
24
+ """
25
+ # Nothing to do if caller didn't pass any files
26
+ if not uploaded_files:
27
+ return
28
+
29
+ # Best-effort cleanup: never crash the app due to deletion failure.
30
+ for f in uploaded_files:
31
+ try:
32
+ # Gradio may pass `None` for an unfilled upload component
33
+ if f is None:
34
+ continue
35
+
36
+ # Gradio File objects expose the local temp path as `.name`
37
+ if not hasattr(f, "name"):
38
+ continue
39
+
40
+ # Delete the temp file if it still exists
41
+ if os.path.exists(f.name):
42
  os.remove(f.name)
43
 
44
+ except Exception:
45
+ # Never fail the entire UI due to a cleanup issue
46
+ pass
47
+
48
+ def _reset_state(_):
49
+ """
50
+ Clear per-session cached inference state when upstream inputs change.
51
+
52
+ Args:
53
+ _ (Any): Placeholder input (e.g., the changed Gradio component value). Not used.
54
+
55
+ Returns:
56
+ None: Used to overwrite `gr.State` so the next Submit recomputes features/predictions.
57
+ """
58
+ # Returning None here is a simple "invalidate cache" signal
59
+ return None
60
+
61
  markdown_text = """
62
  **Feature Category Explanations**
63
 
 
96
 
97
  all_features = pd.read_csv('./data/onconpc_features.csv').drop('Unnamed: 0', axis=1).columns.tolist()
98
 
99
+ def get_preds(patients_file, samples_file, mutations_file, cna_file, tumor_id, state):
 
100
  """
101
+ Run OncoNPC inference from uploaded CSV/TXT inputs and return:
102
+ - top-3 predictions,
103
+ - default explanation plot for the selected sample,
104
+ - updated per-session state for later dropdown interactions.
105
 
106
  Args:
107
+ patients_file (gradio.File | None): Uploaded patient file (CSV/TXT) with required columns `PATIENT_ID`, `SEX`.
108
+ samples_file (gradio.File | None): Uploaded sample file (CSV/TXT) with required columns
109
+ `PATIENT_ID`, `AGE_AT_SEQ_REPORT`, `SAMPLE_ID`.
110
+ mutations_file (gradio.File | None): Uploaded mutations file (CSV/TXT) with required columns
111
+ `Tumor_Sample_Barcode`, `Hugo_Symbol`, `Chromosome`, `Start_Position`, `Reference_Allele`, `Tumor_Seq_Allele2`.
112
+ cna_file (gradio.File | None): Uploaded CNA file (CSV/TXT) with required column `Hugo_Symbol`.
113
+ tumor_id (str | int | None): Sample identifier selected by the user.
114
+ state (dict | None): Per-session cache (from `gr.State`). Expected keys:
115
+ - "features": pd.DataFrame (rows=samples, cols=features)
116
+ - "predictions": pd.DataFrame (rows=samples, cols=cancer types + metadata)
117
+ - "sample_id": current sample being interpreted
118
 
119
  Returns:
120
+ tuple:
121
+ (top3_text: str,
122
+ markdown: gr.Markdown,
123
+ explanation_plot_path: str,
124
+ state: dict)
125
  """
126
+ # ------------------------------------------------------------------
127
+ # 0) Decide whether to reuse cached computations
128
+ # ------------------------------------------------------------------
129
+ # If the user is only changing tumor_id (dropdown) and re-submitting,
130
+ # we can reuse previously computed features/predictions and avoid:
131
+ # - re-reading files
132
+ # - recomputing features
133
+ # - recomputing predictions for all samples
134
+ if state is not None and state.get("features") is not None and state.get("predictions") is not None:
135
+ features = state["features"]
136
+ predictions = state["predictions"]
137
  else:
138
+ # ------------------------------------------------------------------
139
+ # 1) First-run path: validate uploads exist
140
+ # ------------------------------------------------------------------
141
+ if patients_file is None or samples_file is None or cna_file is None or mutations_file is None:
142
+ raise gr.Error("Please make sure all files have been properly uploaded")
143
+
144
+ # ------------------------------------------------------------------
145
+ # 2) Read uploaded files into DataFrames
146
+ # ------------------------------------------------------------------
147
+ # NOTE: Gradio stores uploads as temp files; access path via `.name`.
148
+ if patients_file.name.endswith(".txt"):
149
+ patients_df = pd.read_csv(patients_file.name, sep="\t")
150
+ elif patients_file.name.endswith(".csv"):
151
+ patients_df = pd.read_csv(patients_file.name)
152
+ else:
153
+ raise gr.Error("Patients file must be .txt or .csv")
154
+
155
+ if samples_file.name.endswith(".txt"):
156
+ samples_df = pd.read_csv(samples_file.name, sep="\t")
157
+ elif samples_file.name.endswith(".csv"):
158
+ samples_df = pd.read_csv(samples_file.name) # BUGFIX: was patients_file in original code
159
+ else:
160
+ raise gr.Error("Samples file must be .txt or .csv")
161
+
162
+ if mutations_file.name.endswith(".txt"):
163
+ mutations_df = pd.read_csv(mutations_file.name, sep="\t")
164
+ elif mutations_file.name.endswith(".csv"):
165
+ mutations_df = pd.read_csv(mutations_file.name)
166
+ else:
167
+ raise gr.Error("Mutations file must be .txt or .csv")
168
+
169
+ if cna_file.name.endswith(".txt"):
170
+ cna_df = pd.read_csv(cna_file.name, sep="\t")
171
+ elif cna_file.name.endswith(".csv"):
172
+ cna_df = pd.read_csv(cna_file.name)
173
+ else:
174
+ raise gr.Error("CNA file must be .txt or .csv")
175
+
176
+ # ------------------------------------------------------------------
177
+ # 3) Validate the selected tumor_id
178
+ # ------------------------------------------------------------------
179
+ if tumor_id is None or str(tumor_id).strip() == "":
180
+ raise gr.Error("Tumor Sample ID cannot be empty")
181
+
182
+ # ------------------------------------------------------------------
183
+ # 4) Validate expected columns (fail fast with friendly errors)
184
+ # ------------------------------------------------------------------
185
+ patients_columns = {"PATIENT_ID", "SEX"}
186
+ if not patients_columns.issubset(set(patients_df.columns)):
187
+ raise gr.Error(
188
+ f"Patients File: Expected columns {sorted(patients_columns)}, "
189
+ f"Received {patients_df.columns.to_list()}"
190
+ )
191
+
192
+ samples_columns = {"PATIENT_ID", "AGE_AT_SEQ_REPORT", "SAMPLE_ID"}
193
+ if not samples_columns.issubset(set(samples_df.columns)):
194
+ raise gr.Error(
195
+ f"Samples File: Expected columns {sorted(samples_columns)}, "
196
+ f"Received {samples_df.columns.to_list()}"
197
+ )
198
+
199
+ mutations_columns = {
200
+ "Tumor_Sample_Barcode",
201
+ "Hugo_Symbol",
202
+ "Chromosome",
203
+ "Start_Position",
204
+ "Reference_Allele",
205
+ "Tumor_Seq_Allele2",
206
+ }
207
+ if not mutations_columns.issubset(set(mutations_df.columns)):
208
+ raise gr.Error(
209
+ f"Mutations File: Expected columns {sorted(mutations_columns)}, "
210
+ f"Received {mutations_df.columns.to_list()}"
211
+ )
212
+
213
+ cna_columns = {"Hugo_Symbol"}
214
+ if not cna_columns.issubset(set(cna_df.columns)):
215
+ raise gr.Error(
216
+ f"CNA File: Expected columns {sorted(cna_columns)}, "
217
+ f"Received {cna_df.columns.to_list()}"
218
+ )
219
+
220
+ # ------------------------------------------------------------------
221
+ # 5) Build OncoNPC feature matrix from raw inputs
222
+ # ------------------------------------------------------------------
223
+ # `features` should end up indexed by sample IDs compatible with tumor_id dropdown.
224
+ df_features_genie_final, _ = utils.get_onconpc_features_from_raw_data(
225
+ patients_df,
226
+ samples_df,
227
+ mutations_df,
228
+ cna_df,
229
+ features_onconpc_path="./data/features_onconpc.pkl",
230
+ combined_cohort_age_stats_path="./data/combined_cohort_age_stats.pkl",
231
+ mut_sig_weights_filepath="./data/mutation_signatures/sigProfiler*.csv",
232
+ )
233
+ features = df_features_genie_final
234
+
235
+ # ------------------------------------------------------------------
236
+ # 6) Predict for all samples
237
+ # ------------------------------------------------------------------
238
+ # We assume `xgb_onconpc` is loaded at module scope.
239
+ predictions = utils.get_xgboost_latest_cancer_type_preds(
240
+ xgb_onconpc,
241
+ features,
242
+ cancer_types_to_consider,
243
+ )
244
+
245
+ # ------------------------------------------------------------------
246
+ # 7) Cache these in session state so dropdown interactions work
247
+ # ------------------------------------------------------------------
248
+ state = {"features": features, "predictions": predictions}
249
+
250
+ # ------------------------------------------------------------------
251
+ # 8) Optional: delete temp uploaded files now that they're read into memory
252
+ # ------------------------------------------------------------------
253
+ # This prevents disk bloat on Spaces.
254
+ cleanup_files([patients_file, samples_file, mutations_file, cna_file])
255
+
256
+ # ------------------------------------------------------------------
257
+ # 9) Set the sample being interpreted for this session
258
+ # ------------------------------------------------------------------
259
+ if tumor_id not in state["features"].index:
260
+ raise gr.Error(f"Tumor Sample ID {tumor_id} not found in computed features index.")
261
+ state["sample_id"] = tumor_id
262
+
263
+ # ------------------------------------------------------------------
264
+ # 10) Compute SHAP + generate a plot for the currently selected sample
265
+ # ------------------------------------------------------------------
266
+ # IMPORTANT: Only compute SHAP for one sample to keep this lightweight and fast.
267
+ warnings.filterwarnings("ignore")
268
+ features_one = state["features"].loc[[tumor_id]]
269
+ shaps = utils.obtain_shap_values_with_latest_xgboost(xgb_onconpc, features_one)
270
+
271
+ # Keep predictions consistent with `features_one`
272
+ preds_one = state["predictions"].loc[[tumor_id]]
273
+
274
+ # `results` is a dict keyed by query_id; each entry includes explanation_plot path
275
+ results = utils.get_onconpc_prediction_explanations(
276
+ [tumor_id],
277
+ preds_one,
278
+ shaps,
279
+ features_one,
280
+ cancer_types_to_consider,
281
+ save_plot=True,
282
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
283
 
284
+ # ------------------------------------------------------------------
285
+ # 11) Return UI outputs + updated state
286
+ # ------------------------------------------------------------------
287
+ return (
288
+ get_top3(state["predictions"], tumor_id),
289
+ gr.Markdown(markdown_text),
290
+ results[tumor_id]["explanation_plot"],
291
+ state,
292
+ )
 
 
 
293
 
294
  def parse_inputs(age, gender, CNA_events, mutations):
295
  # Normalization of age input
 
378
  # Return the data as a DataFrame
379
  return pd.DataFrame([data])
380
 
381
+ def get_preds_min_info(age, gender, CNA_events, mutations, state, output="Top Prediction"):
382
  """
383
+ Run OncoNPC inference from manual inputs and return:
384
+ - top-3 predictions,
385
+ - default explanation plot (top prediction) for the manual sample,
386
+ - updated per-session state.
387
 
388
+ Args:
389
+ age (int | float): Age at sequencing/report; standardized using cohort stats.
390
+ gender (str): "Male" or "Female".
391
+ CNA_events (str): CNA event string like "RAF1 2 | PPARG 2".
392
+ mutations (str): Mutation string like "ERBB2, chr17, 378..., C, T | ...".
393
+ state (dict | None): Per-session cache (from `gr.State`). Will be overwritten.
394
+ output (str): Reserved for future output selection. Currently unused.
395
 
396
  Returns:
397
+ tuple:
398
+ (top3_text: str,
399
+ markdown: gr.Markdown,
400
+ explanation_plot_path: str,
401
+ state: dict)
402
  """
403
+ # ------------------------------------------------------------------
404
+ # 1) Parse manual inputs into a single-row feature frame
405
+ # ------------------------------------------------------------------
406
+ # parse_inputs() should produce a DataFrame with one row (the "manual input" sample)
 
407
  features = parse_inputs(age, gender, CNA_events, mutations)
408
+
409
+ # ------------------------------------------------------------------
410
+ # 2) Ensure feature columns match training features exactly (same set + same order)
411
+ # ------------------------------------------------------------------
412
+ all_features_local = (
413
+ pd.read_csv("./data/onconpc_features.csv")
414
+ .drop("Unnamed: 0", axis=1)
415
+ .columns.tolist()
416
+ )
417
+
418
+ # Drop any extra columns not in training feature list
419
+ for c in list(features.columns):
420
+ if c not in all_features_local:
421
+ features = features.drop(c, axis=1)
422
+
423
+ # Add missing columns as zeros
424
+ missing_cols = [c for c in all_features_local if c not in features.columns]
425
+ if missing_cols:
426
+ features = pd.concat([features, pd.DataFrame({c: [0] for c in missing_cols})], axis=1)
427
+
428
+ # Reorder columns to match training
429
+ features = features[all_features_local]
430
+
431
+ # ------------------------------------------------------------------
432
+ # 3) Predict + explain
433
+ # ------------------------------------------------------------------
434
+ predictions = utils.get_xgboost_latest_cancer_type_preds(
435
+ xgb_onconpc, features, cancer_types_to_consider
436
+ )
437
+
438
+ warnings.filterwarnings("ignore")
439
  shaps = utils.obtain_shap_values_with_latest_xgboost(xgb_onconpc, features)
440
 
441
+ # Manual sample id convention: index 0
442
  query_ids = [0]
443
+ results = utils.get_onconpc_prediction_explanations(
444
+ query_ids,
445
+ predictions,
446
+ shaps,
447
+ features,
448
+ cancer_types_to_consider,
449
+ save_plot=True,
450
+ )
451
 
452
+ # ------------------------------------------------------------------
453
+ # 4) Store per-session state used by update_image()
454
+ # ------------------------------------------------------------------
455
+ state = {"features": features, "predictions": predictions, "sample_id": 0}
456
+
457
+ # ------------------------------------------------------------------
458
+ # 5) Return UI outputs + state
459
+ # ------------------------------------------------------------------
460
+ return (
461
+ get_top3(predictions, 0),
462
+ gr.Markdown(markdown_text, elem_classes="markdown-box"),
463
+ results[0]["explanation_plot"],
464
+ state,
465
+ )
466
 
467
  def get_top3(predictions, tumor_sample_id):
468
  """
 
520
  sample_ids = df['SAMPLE_ID'].unique().tolist()
521
  return gr.Dropdown(choices=sample_ids)
522
 
523
+ def update_image(target, state):
524
+ """
525
+ Regenerate the explanation plot for a different target cancer type using cached state.
 
 
 
526
 
527
+ Args:
528
+ target (str): Cancer type selected from dropdown; must be in `cancer_types_to_consider`.
529
+ state (dict | None): Per-session state produced by get_preds()/get_preds_min_info().
530
+
531
+ Returns:
532
+ str: Filesystem path to the newly generated explanation plot image for (sample_id, target).
533
+ """
534
+ # ------------------------------------------------------------------
535
+ # 0) Validate state
536
+ # ------------------------------------------------------------------
537
+ if state is None or state.get("features") is None or state.get("predictions") is None:
538
+ raise gr.Error("Please click Submit first.")
539
+
540
+ features = state["features"]
541
+ predictions = state["predictions"]
542
+ sample_id = state.get("sample_id")
543
+
544
+ if sample_id is None:
545
+ raise gr.Error("Please click Submit first.")
546
+
547
+ if target not in cancer_types_to_consider:
548
+ raise gr.Error(f"Unknown target cancer type: {target}")
549
+
550
+ # ------------------------------------------------------------------
551
+ # 1) Compute SHAP for only the selected sample (fast)
552
+ # ------------------------------------------------------------------
553
+ features_one = features.loc[[sample_id]]
554
+ shaps_cup = utils.obtain_shap_values_with_latest_xgboost(xgb_onconpc, features_one)
555
+
556
+ # In multiclass setting, SHAP is typically indexed by class position
557
+ target_idx = cancer_types_to_consider.index(target)
558
+
559
+ # ------------------------------------------------------------------
560
+ # 2) Build per-feature SHAP vector for this sample and target
561
+ # ------------------------------------------------------------------
562
+ shap_pred_cancer_df = pd.DataFrame(
563
+ shaps_cup[target_idx], # shape ~ (1, n_features) for this single sample
564
+ index=features_one.index, # [sample_id]
565
+ columns=features_one.columns, # feature names
566
+ )
567
  shap_pred_sample_df = shap_pred_cancer_df.loc[sample_id]
568
+ feature_sample_df = features_one.loc[sample_id]
569
+
570
+ # Predicted probability for selected target class
571
  probability = predictions.loc[sample_id][target]
572
+
573
+ # ------------------------------------------------------------------
574
+ # 3) Generate and save plot
575
+ # ------------------------------------------------------------------
576
+ sample_info = f"Prediction: {target}\nPrediction probability: {probability:.3f}"
577
+
578
+ # Partition features into human-friendly groups for the plot legend
579
+ feature_group_to_features_dict, feature_to_feature_group_dict = utils.partition_feature_names_by_group(
580
+ features_one.columns
581
+ )
582
+
583
+ explanation_plot = utils.get_individual_pred_interpretation(
584
+ shap_pred_sample_df,
585
+ feature_sample_df,
586
+ feature_group_to_features_dict,
587
+ feature_to_feature_group_dict,
588
+ sample_info=sample_info,
589
+ filename=f"{sample_id}_{target}_plot",
590
+ filepath="./others_prediction_explanation",
591
+ save_plot=True,
592
+ )
593
+
594
  return explanation_plot
595
 
596
  def show_row(value):
 
624
 
625
  def launch_gradio(server_name, server_port):
626
  """
627
+ Create and launch the Gradio UI, wiring callbacks with per-session `gr.State`
628
+ so interpretation dropdown changes work without recomputing or relying on deleted temp files.
629
 
630
  Args:
631
+ server_name (str): Server interface to bind (e.g., "0.0.0.0" on Spaces).
632
+ server_port (int): Port number to bind.
633
 
634
+ Returns:
635
+ None
636
  """
637
+ manual_input_description = (
638
+ "In this option, you will manually enter patient and tumor data including age at sequencing, sex, "
639
+ "copy number alterations (CNA) events, and somatic mutations."
640
+ )
641
+
642
+ csv_description = (
643
+ "In this option, you can directly upload relevant CSV files, including clinical patient, clinical sample, "
644
+ "mutations, and CNA data from [cBioPortal](https://www.cbioportal.org) or "
645
+ "[AACR GENIE](https://www.aacr.org/professionals/research/aacr-project-genie/). "
646
+ "Please see the example data in this [link](https://github.com/itmoon7/onconpc/tree/main/data/mock_n_3_data) <br> "
647
+ "Note that column names `Tumor_Sample_Barcode` and `SAMPLE_ID` both refer to the sample identifier column and should be the same."
648
+ )
649
+
650
+ with gr.Blocks(
651
+ css=(
652
+ ".markdown-box { border: 1px solid #E0E0E0; border-radius: 8px; "
653
+ "box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05); padding: 15px; margin-bottom: 20px; }"
654
+ )
655
+ ) as demo:
656
+
657
+ # Main selector: choose input mode
658
+ d = gr.Dropdown(
659
+ label="Select your preferred input method for tumor data to visualize predictions made by OncoNPC",
660
+ choices=["Manual Inputs", "CSV File"],
661
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
662
 
663
+ # Helper text rows (toggled with the same dropdown)
664
  with gr.Row(visible=False) as r3:
665
+ gr.Markdown(csv_description, elem_classes="markdown-box")
666
  with gr.Row(visible=False) as r4:
667
+ gr.Markdown(manual_input_description, elem_classes="markdown-box")
668
 
669
+ # ------------------------------------------------------------------
670
+ # CSV MODE UI (files + sample dropdown)
671
+ # ------------------------------------------------------------------
672
  with gr.Row(visible=False) as r1:
673
+ # Per-session cached state for CSV mode.
674
+ # This is what makes dropdown plot changes possible without recomputing.
675
+ state_csv = gr.State(value=None)
676
+
677
  with gr.Column():
678
+ gr.Markdown("**Clinical Patient Data File:** Required columns: `PATIENT_ID`, `SEX`.")
679
  patients_file = gr.File(label="Upload clinical patients data")
680
+
681
+ gr.Markdown("**Clinical Sample Data File:** Required columns: `PATIENT_ID`, `AGE_AT_SEQ_REPORT`, `SAMPLE_ID`.")
682
  samples_file = gr.File(label="Upload clinical samples data")
683
+
684
+ gr.Markdown(
685
+ "**Mutations Data File:** Required columns: `Tumor_Sample_Barcode`, `Hugo_Symbol`, `Chromosome`, "
686
+ "`Start_Position`, `Reference_Allele`, `Tumor_Seq_Allele2`."
687
+ )
688
  mutations_file = gr.File(label="Upload mutations data")
689
+
690
+ gr.Markdown("**CNA Data File:** Required column: `Hugo_Symbol`.")
691
  cna_file = gr.File(label="Upload CNA data")
692
+
693
+ # Sample ID dropdown is populated dynamically when samples_file changes
694
+ tumor_sample_id = gr.Dropdown(choices=[], label="Tumor Sample ID")
695
+
696
+ submit_button_csv = gr.Button("Submit")
697
 
698
  with gr.Column():
699
  predictions_output = gr.Textbox(label="Top 3 Predicted Cancer Types")
700
+ category_explanation = gr.Markdown(markdown_text, elem_classes="markdown-box")
701
+
702
+ image = gr.Image(label="Image Display")
703
+
704
+ # User selects alternate target cancer type for interpretation plot
705
+ output_selector = gr.Dropdown(
706
+ choices=cancer_types_to_consider,
707
+ label="Output Options",
708
+ filterable=True,
709
+ )
710
 
711
+ gr.Markdown(
712
+ "For more details regarding OncoNPC feature processing, OncoNPC performance, and clinical implications of OncoNPC predictions, "
713
+ "please see our [manuscript](https://www.nature.com/articles/s41591-023-02482-6).",
714
+ elem_classes="markdown-box",
715
+ )
716
+
717
+ # When samples file changes, parse IDs and populate tumor_sample_id dropdown
718
  samples_file.change(extract_sample_ids, inputs=samples_file, outputs=tumor_sample_id)
 
 
719
 
720
+ # If any upstream input changes, invalidate cached state (forces recompute on next submit)
721
+ patients_file.change(_reset_state, inputs=patients_file, outputs=state_csv)
722
+ samples_file.change(_reset_state, inputs=samples_file, outputs=state_csv)
723
+ mutations_file.change(_reset_state, inputs=mutations_file, outputs=state_csv)
724
+ cna_file.change(_reset_state, inputs=cna_file, outputs=state_csv)
725
+
726
+ # Submit: computes features/preds (or reuses cached), generates default plot for selected tumor_id,
727
+ # and returns updated state_csv for later dropdown interactions.
728
+ submit_button_csv.click(
729
+ get_preds,
730
+ inputs=[patients_file, samples_file, mutations_file, cna_file, tumor_sample_id, state_csv],
731
+ outputs=[predictions_output, category_explanation, image, state_csv],
732
+ )
733
+
734
+ # Dropdown: regenerates plot using cached state only (no file reads)
735
+ output_selector.change(
736
+ update_image,
737
+ inputs=[output_selector, state_csv],
738
+ outputs=[image],
739
+ )
740
+
741
+ # ------------------------------------------------------------------
742
+ # MANUAL MODE UI (text inputs)
743
+ # ------------------------------------------------------------------
744
  with gr.Row(visible=False) as r2:
745
+ # Per-session cached state for manual mode (separate from CSV mode).
746
+ state_manual = gr.State(value=None)
747
+
748
  with gr.Column():
749
  age = gr.Number(label="Age")
750
  gender = gr.Radio(choices=["Male", "Female"], label="Sex")
 
 
 
 
 
 
 
 
 
751
 
752
+ cna_events = gr.Textbox(
753
+ lines=7,
754
+ placeholder="Copy Number Alterations\n\nExample: RAF1 2 | PPARG 2",
755
+ label="Genes with CNA Events",
756
+ )
757
+
758
+ mutations = gr.Textbox(
759
+ lines=7,
760
+ placeholder=(
761
+ "Somatic Mutations\n\nExample: ERBB2, chr17, 37868208, C, T | "
762
+ "CDKN1A, chr6, 36651880, -, GTCAGAACCGGCTGGGGATGTCC"
763
+ ),
764
+ label="Mutations",
765
+ )
766
+
767
+ submit_button_manual = gr.Button("Submit")
768
 
769
+ with gr.Column():
770
+ predictions_output_m = gr.Textbox(label="Top 3 Predicted Cancer Types")
771
+ category_explanation_m = gr.Markdown(markdown_text, elem_classes="markdown-box")
772
+
773
+ image_m = gr.Image(label="Image Display")
774
+
775
+ output_selector_m = gr.Dropdown(
776
+ choices=cancer_types_to_consider,
777
+ label="Output Options",
778
+ filterable=True,
779
+ )
780
+
781
+ gr.Markdown(
782
+ "For more details regarding OncoNPC feature processing, OncoNPC performance, and clinical implications of OncoNPC predictions, "
783
+ "please see our [manuscript](https://www.nature.com/articles/s41591-023-02482-6).",
784
+ elem_classes="markdown-box",
785
+ )
786
+
787
+ # Manual submit: compute features/preds for a single sample (id 0) and store in state_manual
788
+ submit_button_manual.click(
789
+ get_preds_min_info,
790
+ inputs=[age, gender, cna_events, mutations, state_manual],
791
+ outputs=[predictions_output_m, category_explanation_m, image_m, state_manual],
792
+ )
793
+
794
+ # Manual dropdown: regenerate plot using manual cached state
795
+ output_selector_m.change(
796
+ update_image,
797
+ inputs=[output_selector_m, state_manual],
798
+ outputs=[image_m],
799
+ )
800
+
801
+ # ------------------------------------------------------------------
802
+ # Toggle which UI rows are visible based on input mode dropdown
803
+ # ------------------------------------------------------------------
804
  d.change(show_row, d, [r1, r2])
805
  d.change(show_row, d, [r3, r4])
806
+
807
+ # Launch Gradio app
808
+ demo.launch(debug=True, share=False, server_name=server_name, server_port=server_port)