Add AUC as an additional metric

app.py

@@ -2,7 +2,7 @@ import json
 import os
 import requests
 import gradio as gr
-
+from sklearn.metrics import roc_auc_score
 # -----------------------------
 # 1. Configuration & Data Mapping
 # -----------------------------
@@ -132,6 +132,53 @@ def run_evaluation(cancer_type, strategy):
     | **Actual NO** | {fp} (FP) | {tn} (TN) |
     """
 
+def run_auc_experiment(cancer_type):
+    data_path = CANCER_MAP.get(cancer_type)
+    gt_path = "data/ground_truth_5yr_recurrence.json"
+    
+    with open(data_path, 'r') as f: patient_db = json.load(f)
+    with open(gt_path, 'r') as f: all_gt = json.load(f)
+    
+    gt_labels = all_gt.get(GT_MAP[cancer_type], {})
+    eval_ids = [pid for pid in gt_labels.keys() if pid in patient_db]
+    
+    y_true = [] # Actual labels (1 for Yes, 0 for No)
+    y_scores = [] # Model's predicted probabilities
+    
+    yield f"📊 Starting AUC experiment for {len(eval_ids)} patients..."
+
+    for i, pid in enumerate(eval_ids):
+        # Convert "Yes"/"No" ground truth to 1/0
+        y_true.append(1 if gt_labels[pid] == "Yes" else 0)
+        
+        patient_json = json.dumps(patient_db[pid])
+        
+        # PROMPT CHANGE: Ask for a probability score
+        auc_prompt = [
+            {"role": "system", "content": "You are an oncology expert. Provide a 5-year recurrence probability score from 0 to 100. Respond ONLY with the number (e.g., '75')."},
+            {"role": "user", "content": f"Patient Data: {patient_json}"}
+        ]
+        
+        raw_res = ollama_chat(auc_prompt, temperature=0).strip()
+        
+        # Extract the first number found in the response
+        try:
+            score = float(re.findall(r"(\d+)", raw_res)[0]) / 100.0
+            y_scores.append(score)
+        except:
+            y_scores.append(0.5) # Fallback to neutral if parsing fails
+        
+        if (i + 1) % 5 == 0:
+            yield f"🔄 Processing: {i+1}/{len(eval_ids)}..."
+
+    # Calculate AUC
+    try:
+        auc_value = roc_auc_score(y_true, y_scores)
+        result_md = f"## AUC Result: {cancer_type}\n**Area Under the Curve (AUC):** {auc_value:.4f}"
+    except Exception as e:
+        result_md = f"Error calculating AUC: {e}. Check if you have both classes present in the data."
+
+    yield result_md
 # -----------------------------
 # 4. Helper UI Logic (Chat)
 # -----------------------------
@@ -168,6 +215,14 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
             results_md = gr.Markdown("Select criteria and start to see metrics.")
 
         # TAB 2: Clinical Assistant
+        with gr.TabItem("📈 AUC Experiment"):
+            gr.Markdown("### Calculate AUC using Probability Scores")
+            auc_cancer = gr.Dropdown(label="Cancer Type", choices=list(CANCER_MAP.keys()))
+            auc_btn = gr.Button("Run AUC Test", variant="primary")
+            auc_results = gr.Markdown()
+
+        auc_btn.click(run_auc_experiment, auc_cancer, auc_results)
+
         with gr.TabItem("💬 Clinical Assistant"):
             with gr.Row():
                 with gr.Column(scale=1):