Spaces:
Runtime error
Runtime error
| import logging | |
| import secrets | |
| import tempfile | |
| from functools import wraps | |
| from pathlib import Path | |
| import gradio as gr | |
| import numpy as np | |
| import pandas as pd | |
| from PIL import Image | |
| from core import LABELS, MODEL_ID, REVISION, clean_image, metrics, predict, test_data, test_predictions | |
| def guarded(fn): | |
| def wrapped(*args): | |
| try: | |
| return fn(*args) | |
| except ValueError as error: | |
| raise gr.Error(str(error)) from error | |
| except Exception: | |
| logging.exception("Demo operation failed") | |
| raise gr.Error("Could not load the model or dataset. Check the Space logs or retry shortly.") | |
| return wrapped | |
| def classify(image, truth): | |
| image = clean_image(image) | |
| scores = predict([image])[0] | |
| label = LABELS[int(np.argmax(scores))] | |
| comparison = "No reference label supplied: correctness cannot be assessed." | |
| if truth in LABELS: | |
| comparison = f"{'Matches' if label == truth else 'Does not match'} your supplied reference label ({truth})." | |
| return dict(zip(LABELS, map(float, scores))), ( | |
| f"**Model prediction: {label}.** {comparison}\n\n" | |
| "Scores are uncalibrated softmax outputs, not clinical probabilities. " | |
| "An uploaded image is not verified to be unseen. The model always chooses between two classes, " | |
| "even for unrelated images; it cannot establish that an image is a chest X-ray." | |
| ) | |
| def random_case(): | |
| images, _ = test_data() | |
| idx = secrets.randbelow(len(images)) | |
| return idx, Image.fromarray(images[idx]), None, "Make a guess, then reveal the result.", None | |
| def reveal(idx, guess): | |
| if idx is None: | |
| raise ValueError("Load a test image first.") | |
| _, labels = test_data() | |
| scores = test_predictions()[idx] | |
| truth, predicted = LABELS[int(labels[idx])], LABELS[int(np.argmax(scores))] | |
| student = "No student guess supplied." if guess is None else f"Your guess: {guess} — {'correct' if guess == truth else 'incorrect'}." | |
| return f"**Test image #{idx} · Reference: {truth} · Model: {predicted}**\n\n{student}", dict(zip(LABELS, map(float, scores))) | |
| def pct(value): | |
| return "Not defined" if value is None else f"{value:.1%}" | |
| def evaluate(threshold, mistakes_only): | |
| images, labels = test_data() | |
| probabilities = test_predictions() | |
| m = metrics(labels, probabilities, threshold) | |
| predictions = (probabilities[:, 1] >= threshold).astype(int) | |
| summary = ( | |
| f"### Results on all {m['n']} test images\n" | |
| f"**Accuracy {pct(m['accuracy'])} · Sensitivity {pct(m['sensitivity'])} · " | |
| f"Specificity {pct(m['specificity'])} · Precision {pct(m['precision'])}**\n\n" | |
| f"Pneumonia threshold: **{threshold:.2f}**. Majority-class baseline: {max(np.bincount(labels))/len(labels):.1%}. " | |
| "Sensitivity is the fraction of pneumonia examples detected; specificity is the fraction of normal examples correctly classified. " | |
| "Precision is the fraction of pneumonia predictions matching the reference label.\n\n" | |
| "Changing the threshold explores trade-offs on this test set. Do not tune a final threshold here and then claim an independent test result." | |
| ) | |
| matrix = pd.DataFrame({"Reference label": LABELS, "Predicted Normal": [m['tn'], m['fn']], | |
| "Predicted Pneumonia": [m['fp'], m['tp']]}) | |
| rows = pd.DataFrame({"test_index": range(len(labels)), "reference": [LABELS[i] for i in labels], | |
| "prediction": [LABELS[i] for i in predictions], | |
| "pneumonia_score": probabilities[:, 1], "correct": labels == predictions, | |
| "threshold": threshold, "model_revision": REVISION}) | |
| directory = Path(tempfile.mkdtemp(prefix="medical-eval-")) | |
| csv_path = directory / "test_predictions.csv" | |
| rows.to_csv(csv_path, index=False) | |
| indices = np.flatnonzero(labels != predictions) if mistakes_only else np.arange(len(labels)) | |
| gallery = [(Image.fromarray(images[i]), f"#{i} · True {LABELS[labels[i]]} · Pred {LABELS[predictions[i]]} · P(pneumonia) {probabilities[i,1]:.2f}") | |
| for i in indices[:24]] | |
| return summary, matrix, rows, gallery, str(csv_path) | |
| CSS = """ | |
| .gradio-container {max-width: 1160px !important; margin: auto;} | |
| #hero {padding: 24px; border-radius: 18px; background: linear-gradient(120deg,#102c46,#155e75); color: white; margin-bottom: 16px;} | |
| #hero h1, #hero p {color: white;} | |
| """ | |
| with gr.Blocks(title="MedicalClassifier | Student Lab", delete_cache=(3600, 3600)) as demo: | |
| gr.Markdown("# MedicalClassifier\nExplore what a medical image classifier gets right — and where it fails.", elem_id="hero") | |
| gr.Markdown("**Educational use only — not for diagnosis or treatment.** Use public or de-identified teaching images only. " | |
| "Uploads are processed on this server and may be temporarily cached; do not upload patient identifiers. " | |
| "This model was trained on small pediatric chest X-rays, not validated for general clinical use.") | |
| with gr.Tab("1 · Try an image"): | |
| gr.Markdown("Upload a PNG or JPEG teaching image. An optional reference label lets you compare the prediction; it is not used as model input.") | |
| with gr.Row(): | |
| with gr.Column(): | |
| upload = gr.Image(type="pil", sources=["upload"], label="Teaching image", height=320) | |
| truth = gr.Radio(["Unknown"] + LABELS, value="Unknown", label="Known reference label (optional)") | |
| run = gr.Button("Classify image", variant="primary") | |
| with gr.Column(): | |
| scores = gr.Label(label="Model scores", num_top_classes=2) | |
| result = gr.Markdown() | |
| run.click(classify, [upload, truth], [scores, result], api_name="classify") | |
| upload.change(lambda: (None, "", "Unknown"), outputs=[scores, result, truth]) | |
| truth.change(lambda: (None, ""), outputs=[scores, result]) | |
| with gr.Tab("2 · Guess, then reveal"): | |
| gr.Markdown("Try a random image from the official test split. The reference label stays hidden until you reveal it. " | |
| "Images are only 28 × 28 pixels; the preview enlarges them without adding detail.") | |
| index = gr.State(None) | |
| load = gr.Button("Load random test image", variant="primary") | |
| with gr.Row(): | |
| case = gr.Image(type="pil", interactive=False, height=300, label="Test image") | |
| with gr.Column(): | |
| guess = gr.Radio(LABELS, label="Your guess") | |
| reveal_btn = gr.Button("Reveal reference and model prediction") | |
| feedback = gr.Markdown() | |
| case_scores = gr.Label(label="Model scores") | |
| load.click(random_case, outputs=[index, case, guess, feedback, case_scores]) | |
| reveal_btn.click(reveal, [index, guess], [feedback, case_scores]) | |
| with gr.Tab("3 · Measure performance"): | |
| gr.Markdown("Evaluate **all 624 test images**, not a hand-picked sample. First evaluation downloads the model and dataset and may take a minute. " | |
| "Later evaluations reuse model scores. The single-image tabs use the highest score (threshold 0.5).") | |
| threshold = gr.Slider(0.01, 0.99, value=0.5, step=0.01, label="Pneumonia decision threshold (benchmark only)") | |
| errors = gr.Checkbox(value=True, label="Show only mistakes in the gallery") | |
| evaluate_btn = gr.Button("Evaluate test set", variant="primary") | |
| summary = gr.Markdown() | |
| matrix = gr.Dataframe(interactive=False, label="Confusion matrix · rows = reference, columns = prediction") | |
| gallery = gr.Gallery(label="First 24 matching test images, in dataset order", columns=4, height=450) | |
| rows = gr.Dataframe(interactive=False, label="All test predictions") | |
| download = gr.File(label="Download complete results (CSV)") | |
| evaluate_btn.click(evaluate, [threshold, errors], [summary, matrix, rows, gallery, download], api_name="evaluate") | |
| for control in [threshold, errors]: | |
| control.change(lambda: ("Settings changed. Click Evaluate test set to refresh results.", None, None, [], None), | |
| outputs=[summary, matrix, rows, gallery, download]) | |
| with gr.Tab("About this experiment"): | |
| gr.Markdown(f""" | |
| ### What does “unseen” mean here? | |
| The [published training settings](https://huggingface.co/{MODEL_ID}/blob/{REVISION}/training_params.json) | |
| name `pneumoniamnist:train` and `pneumoniamnist:validation`. This app evaluates only the official **test** split. | |
| That supports treating it as held out from this training run; it does not independently prove absence of all prior exposure or duplicates. | |
| Uploaded images have no verified training provenance. | |
| ### Reproducible inference | |
| [Model](https://huggingface.co/{MODEL_ID}) revision: `{REVISION}`. | |
| The saved image processor performs RGB input conversion, resizing/cropping to 224 × 224 and ImageNet normalization. | |
| No augmentation or learning happens in the app. Class IDs follow the [official MedMNIST mapping](https://github.com/MedMNIST/MedMNIST/blob/main/medmnist/info.py): **0 = Normal, 1 = Pneumonia**. | |
| The [training dataset loader](https://huggingface.co/datasets/galilai-group/medmnist/blob/main/medmnist.py) | |
| uses the same [official archive](https://zenodo.org/records/10519652) as this app. Its checksum is verified before use. | |
| ### Discuss with your class | |
| * Why can a model be very confident and still wrong? | |
| * What happens to false negatives and false positives when the threshold changes? | |
| * Why might adult X-rays, higher-resolution scans or images from another hospital behave differently? | |
| * Why does repeatedly tuning on the test set compromise its role as an independent evaluation? | |
| The labels are dataset annotations, not a medical assessment of the person using this app. | |
| The app cannot detect all unsuitable inputs, and scores have not been calibrated. | |
| No uploaded image is added to the benchmark, used for training, or deliberately logged by this app. | |
| Gradio's temporary file cache is cleaned periodically (hourly, for files at least one hour old). | |
| Dataset: **PneumoniaMNIST / MedMNIST v2**, CC BY 4.0. | |
| Credit: Yang et al., *MedMNIST v2 — A large-scale lightweight benchmark for 2D and 3D biomedical image classification*, Scientific Data (2023). | |
| [Dataset documentation](https://medmnist.com/). | |
| """) | |
| if __name__ == "__main__": | |
| demo.queue(default_concurrency_limit=1).launch(theme=gr.themes.Soft(primary_hue="teal"), css=CSS, | |
| max_file_size="10mb", show_error=False) | |