Create app/gemini_scanner.py

app/gemini_scanner.py

@@ -0,0 +1,262 @@
+"""
+Gemini Vision API Integration for Image Scanning
+Provides OCR and product identification for accessibility
+"""
+
+import base64
+from typing import Dict, Optional
+from pydantic import BaseModel
+
+GEMINI_API_KEY = "AIzaSyCKg9oniH4PH6gLcG7WZzrjNAESqmZCk2o"
+GEMINI_API_URL = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-exp:generateContent?key={GEMINI_API_KEY}"
+
+class ScanResult(BaseModel):
+    success: bool
+    product_type: str  # "medication", "grocery", "receipt", "unknown"
+    product_name: Optional[str] = None
+    brand: Optional[str] = None
+    details: Dict = {}
+    spoken_description: str
+    warnings: list = []
+
+async def scan_product_image(image_data: str, scan_type: str = "auto") -> ScanResult:
+    """
+    Scan a product image using Gemini Vision API
+    
+    Args:
+        image_data: Base64 encoded image
+        scan_type: "medication", "grocery", "receipt", or "auto"
+    
+    Returns:
+        ScanResult with all extracted information
+    """
+    
+    # Different prompts based on scan type
+    prompts = {
+        "medication": """
+Analyze this medication/pharmaceutical product image carefully for a visually impaired user.
+
+Provide COMPLETE information in this EXACT format:
+
+PRODUCT TYPE: [Medication/Prescription/OTC]
+PRODUCT NAME: [Full medication name]
+BRAND: [Brand name if visible]
+GENERIC NAME: [Generic name if visible]
+STRENGTH: [Dosage strength, e.g., 500mg, 10mg]
+FORM: [Tablet/Capsule/Liquid/Cream/etc]
+QUANTITY: [Number of pills/volume]
+EXPIRATION DATE: [Date if visible - CHECK CAREFULLY]
+NDC/BARCODE: [If visible]
+PRESCRIPTION INFO: [Patient name, Rx number if visible]
+DIRECTIONS: [Usage instructions]
+WARNINGS: [Any warning labels - list ALL you see]
+INTERACTIONS: [Drug interaction warnings if visible]
+
+SPOKEN DESCRIPTION: [Create a 2-3 sentence natural description that would be read aloud to someone who cannot see the image. Include the most critical safety information.]
+
+SAFETY ALERTS: [If expired or any critical warnings, mention them prominently]
+
+Be thorough and accurate. If you cannot read something clearly, say "Not clearly visible" rather than guessing.
+""",
+        
+        "grocery": """
+Analyze this grocery/food product image for a visually impaired user.
+
+Provide COMPLETE information in this EXACT format:
+
+PRODUCT TYPE: [Food category]
+PRODUCT NAME: [Full product name]
+BRAND: [Brand name]
+SIZE/WEIGHT: [Package size]
+PRICE: [If visible on label or tag]
+EXPIRATION/BEST BY: [Date if visible]
+NUTRITIONAL HIGHLIGHTS: [Key nutrition facts if visible - calories, protein, etc]
+INGREDIENTS: [Main ingredients if visible]
+SPECIAL LABELS: [Organic, Gluten-Free, Non-GMO, etc]
+BARCODE/UPC: [If visible]
+
+SPOKEN DESCRIPTION: [Create a natural 2-3 sentence description for someone who cannot see. Include product name, size, and any important details like expiration or special features.]
+
+If anything is unclear, say "Not clearly visible" rather than guessing.
+""",
+        
+        "receipt": """
+Analyze this receipt image for a visually impaired user who wants to track their purchases.
+
+Provide information in this EXACT format:
+
+STORE: [Store name]
+DATE: [Purchase date]
+TIME: [Purchase time if visible]
+TOTAL: [Total amount]
+
+ITEMS PURCHASED: [List each item with price in this format]
+- Item name | Price
+- Item name | Price
+
+PAYMENT METHOD: [If visible]
+
+SPOKEN DESCRIPTION: [Create a natural summary: "Receipt from [Store] on [Date]. Total: [Amount]. You purchased [X] items including..." List 3-5 main items.]
+
+List items clearly so they can be added to inventory.
+""",
+        
+        "auto": """
+Analyze this image and determine what type of product it shows (medication, grocery item, receipt, or other).
+
+Then provide detailed information following these guidelines:
+
+1. First, identify: Is this a MEDICATION, GROCERY ITEM, RECEIPT, or OTHER?
+
+2. For MEDICATIONS: Include name, dosage, warnings, expiration date, usage instructions
+3. For GROCERY: Include product name, brand, size, price, nutritional info, expiration
+4. For RECEIPTS: Include store, date, items purchased, total
+5. For OTHER: Describe what you see
+
+PRODUCT TYPE: [medication/grocery/receipt/other]
+
+[Then provide all relevant details in clear format]
+
+SPOKEN DESCRIPTION: [Create a natural 2-3 sentence description that would be helpful for someone who cannot see the image. Focus on the most important information they would want to know.]
+
+Always prioritize clarity and accuracy for accessibility purposes.
+"""
+    }
+    
+    prompt = prompts.get(scan_type, prompts["auto"])
+    
+    try:
+        import httpx
+        
+        # Prepare the request
+        payload = {
+            "contents": [{
+                "parts": [
+                    {"text": prompt},
+                    {
+                        "inline_data": {
+                            "mime_type": "image/jpeg",
+                            "data": image_data
+                        }
+                    }
+                ]
+            }],
+            "generationConfig": {
+                "temperature": 0.2,  # Low temperature for accuracy
+                "maxOutputTokens": 2000
+            }
+        }
+        
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.post(GEMINI_API_URL, json=payload)
+            response.raise_for_status()
+            
+            data = response.json()
+            analysis_text = data["candidates"][0]["content"]["parts"][0]["text"]
+            
+            # Parse the response
+            result = parse_gemini_response(analysis_text, scan_type)
+            return result
+            
+    except Exception as e:
+        return ScanResult(
+            success=False,
+            product_type="error",
+            spoken_description=f"Sorry, I encountered an error scanning the image: {str(e)}",
+            warnings=["Scan failed - please try again or manually enter information"]
+        )
+
+def parse_gemini_response(text: str, scan_type: str) -> ScanResult:
+    """Parse Gemini's response into structured data"""
+    
+    lines = text.split('\n')
+    details = {}
+    product_name = None
+    brand = None
+    spoken_desc = ""
+    warnings = []
+    detected_type = scan_type if scan_type != "auto" else "unknown"
+    
+    for line in lines:
+        line = line.strip()
+        
+        # Extract key-value pairs
+        if ':' in line:
+            key, value = line.split(':', 1)
+            key = key.strip().upper()
+            value = value.strip()
+            
+            if 'PRODUCT TYPE' in key:
+                detected_type = value.lower()
+                if 'medication' in detected_type or 'prescription' in detected_type:
+                    detected_type = "medication"
+                elif 'grocery' in detected_type or 'food' in detected_type:
+                    detected_type = "grocery"
+                elif 'receipt' in detected_type:
+                    detected_type = "receipt"
+                    
+            elif 'PRODUCT NAME' in key or 'MEDICATION NAME' in key:
+                product_name = value
+                details['name'] = value
+                
+            elif 'BRAND' in key:
+                brand = value
+                details['brand'] = value
+                
+            elif 'SPOKEN DESCRIPTION' in key:
+                spoken_desc = value
+                
+            elif 'WARNING' in key or 'SAFETY ALERT' in key:
+                if value and value != "None" and value != "Not visible":
+                    warnings.append(value)
+                    
+            elif 'EXPIRATION' in key or 'EXPIRES' in key or 'BEST BY' in key:
+                details['expiration'] = value
+                # Check if expired
+                if 'expired' in value.lower():
+                    warnings.append(f"⚠️ EXPIRED: {value}")
+                    
+            elif value and value not in ["None", "Not visible", "Not clearly visible"]:
+                # Store other details
+                clean_key = key.lower().replace(' ', '_')
+                details[clean_key] = value
+    
+    # If no spoken description was found, create one from details
+    if not spoken_desc and product_name:
+        if detected_type == "medication":
+            spoken_desc = f"This is {product_name}"
+            if 'strength' in details:
+                spoken_desc += f", {details['strength']}"
+            if warnings:
+                spoken_desc += ". " + warnings[0]
+        elif detected_type == "grocery":
+            spoken_desc = f"This is {product_name}"
+            if brand:
+                spoken_desc += f" by {brand}"
+            if 'size/weight' in details:
+                spoken_desc += f", {details['size/weight']}"
+        else:
+            spoken_desc = f"Product identified: {product_name or 'Unknown'}"
+    
+    return ScanResult(
+        success=True,
+        product_type=detected_type,
+        product_name=product_name,
+        brand=brand,
+        details=details,
+        spoken_description=spoken_desc or "Image scanned successfully. Please check details.",
+        warnings=warnings
+    )
+```
+
+5. **Click "Commit new file to main"**
+
+---
+
+### **Step 2: Update requirements.txt**
+
+1. **Click on `requirements.txt`**
+2. **Click "Edit"**
+3. **Add this line:**
+```
+httpx==0.25.0