Eckley - Final submission

app.py

@@ -1,142 +1,36 @@
 import gradio as gr
-from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
-import torch
-import pypdf
-import re
+from PIL import Image
+from transformers import BlipProcessor, BlipForConditionalGeneration
 
-# NER Model: Standard setup with aggregation
-skill_pipe = pipeline(
-    "token-classification", 
-    model="ihk/skillner", 
-    aggregation_strategy="simple"
-)
-
-# LLM: Flan-T5-Large
-model_id = "google/flan-t5-large"
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = AutoModelForSeq2SeqLM.from_pretrained(model_id)
-
-def extract_text_from_pdf(file_path):
-    text = ""
-    try:
-        with open(file_path, "rb") as f:
-            reader = pypdf.PdfReader(f)
-            for page in reader.pages:
-                content = page.extract_text()
-                if content:
-                    text += content + "\n"
-    except Exception:
-        return ""
-    return text
-
-def analyze_profile(file_obj, text_input):
-    resume_text = ""
-    if file_obj is not None:
-        if file_obj.name.endswith('.pdf'):
-            resume_text = extract_text_from_pdf(file_obj.name)
-        else:
-            with open(file_obj.name, "r", encoding="utf-8") as f:
-                resume_text = f.read()
-    else:
-        resume_text = text_input
-
-    if not resume_text.strip():
-        return "Please provide a resume.", "N/A"
-
-    # Step 1: Skill Extraction & Cleaning
-    try:
-        results = skill_pipe(resume_text[:2000]) 
-        raw_skills = [res['word'].strip() for res in results if 'SKILL' in res['entity_group']]
-        
-        clean_skills = []
-        for s in raw_skills:
-            s = s.replace("##", "")
-            # Removing non-alphabetic noise
-            s = re.sub(r'^[^a-zA-Z]+|[^a-zA-Z]+$', '', s)
-            if len(s) > 2:
-                clean_skills.append(s)
-        
-        skills = sorted(list(set(clean_skills)))
-    except Exception:
-        return "Extraction failed.", "N/A"
+MODEL_NAME = "Salesforce/blip-image-captioning-base"
 
-    if not skills:
-        return "No skills detected.", "Try a more detailed resume."
+processor = BlipProcessor.from_pretrained(MODEL_NAME)
+model = BlipForConditionalGeneration.from_pretrained(MODEL_NAME)
 
-    skills_str = ", ".join(skills)
+def generate_caption(image):
+    if image is None:
+        return "Please upload an image."
 
-    # Step 2: Few-Shot Prompting
+    if image.mode != "RGB":
+        image = image.convert("RGB")
 
-    few_shot_prompt = (
-        "Example:\n"
-        "Skills: Python, SQL, AWS, Spark\n"
-        "Certifications:\n"
-        "1. AWS Certified Solutions Architect - Professional\n"
-        "2. Databricks Certified Data Engineer Professional\n"
-        "3. Google Professional Data Engineer\n\n"
-        f"Actual Task:\n"
-        f"Skills: {skills_str}\n"
-        "Certifications:\n"
-        "1."
-    )
-    
-    inputs = tokenizer(few_shot_prompt, return_tensors="pt", truncation=True, max_length=512)
-    
+    inputs = processor(image, return_tensors="pt")
 
-    outputs = model.generate(
-        **inputs, 
-        max_new_tokens=80, 
-        num_beams=4,                 # Beam search is logical/less random
-        repetition_penalty=1.2,      # Lower penalty avoids hallucinations
-        no_repeat_ngram_size=2,      # Prevents word-looping
-        early_stopping=True
+    output = model.generate(
+        **inputs,
+        max_new_tokens=30
     )
-    
-    roadmap = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    
-    # Cleaning the output to ensure it starts with our '1.'
-    if not roadmap.startswith("1."):
-        roadmap = "1. " + roadmap
-
-    return skills_str, roadmap
-
-# Documentation 
-arch_doc = """
-### System Architecture
-This system utilizes a three-stage pipeline for document intelligence. I integrated pypdf for raw document ingestion, followed by a JobBERT model fine-tuned for professional skill extraction. For the final reasoning layer, I used Flan-T5-Large to perform a gap analysis and generate an upskilling roadmap.
-"""
 
-eval_doc = """
-### Challenges & Solutions
-Fragmentation: To fix sub-word tokenization (e.g., "Py" + "##spark"), I implemented an Aggregation Strategy and custom Regex cleaning.
-Hallucinations: Early testing showed model "parrotting." I solved this using Few-Shot In-Context Learning and Beam Search (num_beams = 4‬) to ensure factual certification names.
-"""
+    caption = processor.decode(output[0], skip_special_tokens=True)
+    return caption
 
-ethics_doc = """
-### Responsible AI & Ethics
-Bias Mitigation: The model is "feature-restricted"—the LLM only sees technical skills, blinding it to demographic data like names or gender.
-Data Privacy: This application uses transient processing; all document data exists only in volatile memory and is discarded immediately after the session ends.
-"""
-
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
-    gr.Markdown("# Advanced Resume Intelligence")
-    
-    with gr.Row():
-        with gr.Column():
-            file_in = gr.File(label="Upload Resume (PDF/TXT)", file_types=[".pdf", ".txt"])
-            txt_in = gr.Textbox(label="Or Paste Text", lines=5)
-            btn = gr.Button("Analyze Profile", variant="primary")
-        
-        with gr.Column():
-            skill_out = gr.Textbox(label="Identified Skills")
-            advice_out = gr.Textbox(label="AI Training Roadmap")
-            
-    btn.click(fn=analyze_profile, inputs=[file_in, txt_in], outputs=[skill_out, advice_out])
-
-    with gr.Accordion("Project Documentation", open=False):
-        gr.Markdown(arch_doc)
-        gr.Markdown(eval_doc)
-        gr.Markdown(ethics_doc)
+demo = gr.Interface(
+    fn=generate_caption,
+    inputs=gr.Image(type="pil", label="Upload an Image"),
+    outputs=gr.Textbox(label="Generated Caption"),
+    title="AI Image Captioning System",
+    description="Upload an image and generate a caption using Salesforce/blip-image-captioning-base."
+)
 
 if __name__ == "__main__":
     demo.launch()