Thomas - Final submission

app.py

@@ -1,96 +1,136 @@
-import os
-import random
 
 import gradio as gr
-import torch
-from datasets import load_dataset
-from transformers import BlipForQuestionAnswering, BlipProcessor
-
-# Always use BlipProcessor + BlipForQuestionAnswering directly.
-# The Transformers pipeline API for BLIP-VQA changed between v4 and v5 and can echo
-# the question instead of returning the answer. Direct model calls are stable across versions.
-MODEL_ID = "Salesforce/blip-vqa-base"
-
-_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-_processor = BlipProcessor.from_pretrained(MODEL_ID)
-_model = BlipForQuestionAnswering.from_pretrained(MODEL_ID).to(_device)
-_model.eval()
-
-
-def run_vqa(image, question: str) -> str:
-    inputs = _processor(images=image, text=question, return_tensors="pt").to(_device)
-    with torch.no_grad():
-        out = _model.generate(**inputs, max_new_tokens=32)
-    return _processor.decode(out[0], skip_special_tokens=True).strip()
-
-FOOD101_ID = "ethz/food101"
-DEFAULT_VQA_QUESTION = "What kind of food is in this image?"
-
-
-def answer_question(image, question):
-    if not question:
-        return "Please type a question about the image."
-    if image is None:
-        return "Please upload an image (or use Load random sample) before clicking Answer."
-    return run_vqa(image, question)
-
-
-def load_food101_sample(cache: dict | None):
-    """
-    Pull one random example from Hugging Face Datasets (Food-101).
-    Dataset card: https://huggingface.co/datasets/ethz/food101
-    """
-    if cache is None:
-        cache = {"ds": load_dataset(FOOD101_ID, split="validation")}
-    ds = cache["ds"]
-    i = random.randint(0, len(ds) - 1)
-    ex = ds[i]
-    names = ds.features["label"].names
-    gold = names[ex["label"]]
-    note = (
-        f"Loaded from Hugging Face Datasets `{FOOD101_ID}` "
-        f"(split `validation`, index {i}). Reference label: `{gold}`."
-    )
-    return ex["image"], DEFAULT_VQA_QUESTION, cache, note
+from transformers import pipeline
+from PIL import Image
+import numpy as np
 
+# Load Models (Explicit)
+classifier = pipeline(
+    "image-classification",
+    model="google/vit-base-patch16-224",
+    use_fast=True
+)
 
-with gr.Blocks() as demo:
-    ds_cache = gr.State(None)
-    gr.Markdown(
-        "## BLIP Visual Question Answering\n"
-        "Gradio app for [Hugging Face Spaces](https://huggingface.co/docs/hub/spaces). "
-        "Upload your own image **or** load a sample image from "
-        f"[Hugging Face Datasets — `{FOOD101_ID}`](https://huggingface.co/datasets/ethz/food101)."
-    )
-    dataset_note = gr.Markdown("")
-    with gr.Row():
-        image_in = gr.Image(type="pil", label="Image")
-        question_in = gr.Textbox(
-            label="Question",
-            placeholder="e.g. What is the weather in this image?",
+captioner = pipeline(
+    "image-text-to-text",
+    model="Salesforce/blip-image-captioning-base"
+)
+
+sentiment = pipeline(
+    "sentiment-analysis",
+    model="distilbert-base-uncased-finetuned-sst-2-english"
+)
+
+# Smart Matching Function
+def check_match(label, caption):
+    label = label.lower()
+    caption = caption.lower()
+
+    keyword_map = {
+        "dog": ["dog", "puppy"],
+        "cat": ["cat", "kitten"],
+        "car": ["car", "vehicle"],
+        "pizza": ["pizza"],
+        "bird": ["bird"],
+        "person": ["person", "man", "woman", "people"],
+        "bicycle": ["bike", "bicycle"],
+        "laptop": ["laptop", "computer"],
+        "beach": ["beach", "ocean", "sea"],
+        "mountain": ["mountain"]
+    }
+
+    for key, words in keyword_map.items():
+        if key in label:
+            for word in words:
+                if word in caption:
+                    return True
+
+    return False
+
+# Image Processing Function
+def process_image(image):
+    try:
+
+        # Image is already PIL — no conversion needed
+
+        # 1. Classification
+        class_result = classifier(image)
+        top_class = class_result[0]['label']
+        confidence = round(class_result[0]['score'], 3)
+
+        # 2. Captioning (FIXED)
+        print(f"Debug: Type of image for captioner: {type(image)}")
+        print(f"Debug: Image info (if PIL): {image.format, image.size, image.mode}")
+        # Pass image as a list and provide a default text prompt
+        caption_result = captioner(images=[image], text="a photo of")
+
+        caption = caption_result[0]['generated_text']
+
+        # Handle empty caption
+        if not caption or len(caption.strip()) == 0:
+            sentiment_label = "N/A"
+            sentiment_score = 0.0
+            comparison_text = "No Caption ❌"
+            caption = "No caption generated."
+        else:
+            # 3. Sentiment
+            sentiment_result = sentiment(caption)[0]
+            sentiment_label = sentiment_result['label']
+            sentiment_score = round(sentiment_result['score'], 3)
+
+            # 4. Comparison
+            match = top_class.lower() in caption.lower()
+            comparison_text = "Match ✅" if match else "No Match ❌"
+
+        return (
+            f"{top_class} (Confidence: {confidence})",
+            caption,
+            f"{sentiment_label} (Score: {sentiment_score})",
+            comparison_text
         )
-    answer_out = gr.Textbox(label="Answer")
-    with gr.Row():
-        submit_btn = gr.Button("Answer", variant="primary")
-        sample_btn = gr.Button(f"Load random sample ({FOOD101_ID})")
-
-    submit_btn.click(answer_question, [image_in, question_in], [answer_out])
-    sample_btn.click(
-        load_food101_sample,
-        [ds_cache],
-        [image_in, question_in, ds_cache, dataset_note],
+
+    except Exception as e:
+        return (
+            "Error in classification",
+            "Error in captioning",
+            "Error in sentiment",
+            str(e)
+        )
+
+# Create a dummy image for testing the pipelines
+dummy_image_path = "dummy_image.png"
+img_data = np.zeros((100, 100, 3), dtype=np.uint8) # 100x100 black image
+dummy_img = Image.fromarray(img_data)
+dummy_img.save(dummy_image_path)
+
+# Load the dummy image for testing
+img = Image.open(dummy_image_path)
+
+classifier_test_result = classifier(img)
+print("Classifier test result:", classifier_test_result)
+
+# Pass image as a list and provide a default text prompt for testing
+captioner_test_result = captioner(images=[img], text="a photo of")
+print("Captioner test result:", captioner_test_result)
+
+# Gradio Interface
+interface = gr.Interface(
+    fn=process_image,
+    inputs=gr.Image(type="pil"),
+    outputs=[
+        gr.Text(label="Image Classification"),
+        gr.Text(label="Generated Caption"),
+        gr.Text(label="Caption Sentiment"),
+        gr.Text(label="Model Agreement")
+    ],
+    title="Multimodal Image Analysis System",
+    description=(
+        "Upload an image to analyze it using multiple AI models. "
+        "This app compares image classification and captioning results "
+        "and applies NLP to evaluate the caption."
     )
+)
 
+# Launch App
 if __name__ == "__main__":
-    # Hugging Face Spaces sets PORT; bind 0.0.0.0 so the Space proxy can reach the app.
-    # Locally: open http://127.0.0.1:<PORT> (default PORT 7860 if unset).
-    _port = int(os.environ.get("PORT", "7860"))
-    # Queue mode is recommended for Spaces and tends to play nicer with Gradio’s async stack.
-    demo.queue()
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=_port,
-        # Space URL is already public; share=True would add a separate *.gradio.live tunnel (not needed).
-        share=False,
-        show_error=True,
-    )
+    interface.launch(debug=True)