Update app.py

app.py

@@ -19,14 +19,52 @@ from torchvision import models, transforms
 import matplotlib.pyplot as plt
 import seaborn as sns
 from collections import Counter
+import os
+import pickle
 
 # Set up logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Define mean_absolute_percentage_error function
-def mean_absolute_percentage_error(y_true, y_pred):
-    y_true, y_pred = np.array(y_true), np.array(y_pred)
-    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+# Cache file to store extracted text
+CACHE_FILE = "image_text_cache.pkl"
+
+# Load cache if it exists
+if os.path.exists(CACHE_FILE):
+    with open(CACHE_FILE, "rb") as f:
+        cache = pickle.load(f)
+else:
+    cache = {}
+
+# Function to resize images
+def resize_image(image, max_size=(800, 600)):
+    """Resize an image to the specified maximum size."""
+    image.thumbnail(max_size)
+    return image
+
+# Function to extract text from an image
+def extract_text_from_image(image_url):
+    """Extract text from an image using OCR."""
+    if image_url in cache:
+        return cache[image_url]  # Return cached text if available
+    if not image_url or not isinstance(image_url, str) or not image_url.startswith(('http://', 'https://')):
+        return ""  # Skip invalid URLs
+    try:
+        response = requests.get(image_url)
+        response.raise_for_status()  # Raise an error for bad responses (4xx, 5xx)
+        image = Image.open(BytesIO(response.content))
+        image = resize_image(image)  # Resize the image
+        text = pytesseract.image_to_string(image)
+        cache[image_url] = text  # Cache the extracted text
+        return text
+    except Exception as e:
+        logging.error(f"Error extracting text from image: {e}")
+        return ""
+
+# Function to process new images
+def process_new_images(new_images):
+    """Process new images and return a DataFrame with extracted text."""
+    new_images['image_text'] = new_images['image_url'].apply(extract_text_from_image)
+    return new_images
 
 # Load engagement_metrics.json (your company's data)
 logging.info("Loading your company's engagement metrics...")
@@ -121,33 +159,30 @@ logging.info("Combining your company's data and competitor data for model traini
 combined_df = pd.concat([your_df, competitor_df], ignore_index=True)
 
 # Handle missing or invalid image URLs
-combined_df['image_url'] = combined_df['image_url'].fillna('')  # Replace NaN with empty string
+combined_df['image_url'] = combined_df['image_url'].fillna('')
 
-# Extract text from images using OCR
-def extract_text_from_image(image_url):
-    """Extract text from an image using OCR."""
-    if not image_url or not isinstance(image_url, str) or not image_url.startswith(('http://', 'https://')):
-        # Skip empty or invalid URLs
-        return ""
-    try:
-        response = requests.get(image_url)
-        response.raise_for_status()  # Raise an error for bad responses (4xx, 5xx)
-        image = Image.open(BytesIO(response.content))
-        text = pytesseract.image_to_string(image)
-        return text
-    except Exception as e:
-        logging.error(f"Error extracting text from image: {e}")
-        return ""
+# Process only new images (those not in the cache)
+logging.info("Extracting text from new images...")
+new_images = combined_df[~combined_df['image_url'].isin(cache.keys())]
+new_images = process_new_images(new_images)
 
-logging.info("Extracting text from images...")
-combined_df['image_text'] = combined_df['image_url'].apply(extract_text_from_image)
+# Update the combined DataFrame with extracted text
+combined_df.update(new_images)
+
+# Save the updated dataset
+combined_df.to_csv('data_with_extracted_text.csv', index=False)
+
+# Save the cache
+with open(CACHE_FILE, "wb") as f:
+    pickle.dump(cache, f)
+
+logging.info("Incremental processing complete!")
 
 # Analyze image content using a pre-trained model
 def analyze_image(image_url):
     """Analyze image content using a pre-trained model."""
     if not image_url or not isinstance(image_url, str) or not image_url.startswith(('http://', 'https://')):
-        # Skip empty or invalid URLs
-        return None
+        return None  # Skip invalid URLs
     try:
         response = requests.get(image_url)
         response.raise_for_status()  # Raise an error for bad responses (4xx, 5xx)