Update app.py

app.py

@@ -22,7 +22,6 @@ import seaborn as sns
 from collections import Counter
 import pickle
 from transformers import ResNetForImageClassification
-from datasets import load_dataset, Dataset  # Hugging Face datasets library
 
 # Set up logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -35,35 +34,13 @@ os.chdir(WORKING_DIR)
 # Verify the current directory
 logging.info(f"Current working directory: {os.getcwd()}")
 
-# Hugging Face dataset name for persistent storage
-DATASET_NAME = "Fred808/data"  # Replace with your Hugging Face dataset name
+# Cache file to store extracted text
+CACHE_FILE = os.path.join(WORKING_DIR, "image_text_cache.pkl")
 
-# Function to save data to Hugging Face Hub
-def save_to_hub(data, dataset_name):
-    """Save data to Hugging Face Hub as a dataset."""
-    try:
-        dataset = Dataset.from_dict(data)
-        dataset.push_to_hub(dataset_name)
-        logging.info(f"Data saved to Hugging Face Hub: {dataset_name}")
-    except Exception as e:
-        logging.error(f"Error saving data to Hugging Face Hub: {e}")
-
-# Function to load data from Hugging Face Hub
-def load_from_hub(dataset_name):
-    """Load data from Hugging Face Hub."""
-    try:
-        dataset = load_dataset(dataset_name)
-        logging.info(f"Data loaded from Hugging Face Hub: {dataset_name}")
-        return dataset["train"].to_dict()  # Convert to dictionary
-    except Exception as e:
-        logging.error(f"Error loading data from Hugging Face Hub: {e}")
-        return None
-
-# Load cache from Hugging Face Hub
-CACHE_KEY = "image_text_cache"
-cache = load_from_hub(DATASET_NAME)
-if cache and CACHE_KEY in cache:
-    cache = cache[CACHE_KEY]
+# Load cache if it exists
+if os.path.exists(CACHE_FILE):
+    with open(CACHE_FILE, "rb") as f:
+        cache = pickle.load(f)
 else:
     cache = {}
 
@@ -118,22 +95,6 @@ except FileNotFoundError:
     logging.error("solved.json not found. Please ensure the file exists.")
     exit()
 
-# Load competitor data from JSON
-logging.info("Loading competitor data from JSON...")
-try:
-    with open('competitors_data.json', 'r') as f:
-        competitor_data = json.load(f)
-    # Combine all competitors' data into a single DataFrame
-    competitor_dfs = []
-    for competitor, posts in competitor_data.items():
-        df = pd.json_normalize(posts)
-        df['competitor'] = competitor  # Add competitor name for tracking
-        competitor_dfs.append(df)
-    competitor_df = pd.concat(competitor_dfs, ignore_index=True)
-except FileNotFoundError:
-    logging.error("competitors_data.json not found. Please ensure the file exists.")
-    exit()
-
 # Ensure required columns exist in your company's data
 required_columns = ['likes', 'comments', 'shares', 'posting_time', 'caption', 'hashtags']
 missing_columns = [col for col in required_columns if col not in your_df.columns]
@@ -149,23 +110,6 @@ if missing_columns:
             your_df[col] = [[] for _ in range(len(your_df))]  # Fill with default value (list of empty lists)
     logging.info("Default values added for missing columns.")
 
-# Ensure required columns exist in competitor data
-required_columns = ['caption', 'hashtags', 'likes', 'comments', 'date', 'image_url']
-missing_columns = [col for col in required_columns if col not in competitor_df.columns]
-
-if missing_columns:
-    logging.warning(f"Missing required columns in competitor data: {missing_columns}")
-    for col in missing_columns:
-        if col == 'caption':
-            competitor_df[col] = ''  # Fill with default value (empty string)
-        elif col == 'hashtags':
-            competitor_df[col] = [[] for _ in range(len(competitor_df))]  # Fill with default value (list of empty lists)
-        elif col == 'image_url':
-            competitor_df[col] = ''  # Fill with default value (empty string)
-        else:
-            competitor_df[col] = 0  # Fill with default value (integer)
-    logging.info("Default values added for missing columns.")
-
 # Process your company's data
 logging.info("Processing your company's data...")
 your_df['posting_time'] = pd.to_datetime(your_df['posting_time'], format='%Y-%m-%d %H:%M:%S', errors='coerce')
@@ -176,33 +120,20 @@ your_df['hashtag_count'] = your_df['hashtags'].apply(len)
 your_df['caption_sentiment'] = your_df['caption'].apply(lambda x: TextBlob(x).sentiment.polarity)
 your_df['sentiment'] = your_df['caption_sentiment']
 
-# Process competitor data
-logging.info("Processing competitor data...")
-competitor_df['posting_time'] = pd.to_datetime(competitor_df['date'], format='%Y-%m-%d %H:%M:%S', errors='coerce')
-competitor_df = competitor_df[competitor_df['posting_time'].notna()]
-competitor_df['engagement_rate'] = competitor_df['likes'] + competitor_df['comments']
-competitor_df['caption_length'] = competitor_df['caption'].apply(len)
-competitor_df['hashtag_count'] = competitor_df['hashtags'].apply(len)
-competitor_df['caption_sentiment'] = competitor_df['caption'].apply(lambda x: TextBlob(x).sentiment.polarity)
-competitor_df['sentiment'] = competitor_df['caption_sentiment']
-
-# Combine your company's data and competitor data for model training
-logging.info("Combining your company's data and competitor data for model training...")
-combined_df = pd.concat([your_df, competitor_df], ignore_index=True)
-
-# Handle missing or invalid image URLs
-combined_df['image_url'] = combined_df['image_url'].fillna('')
+# Handle missing or invalid image URLs in your company's data
+your_df['image_url'] = your_df['image_url'].fillna('')
 
 # Process only new images (those not in the cache)
 logging.info("Extracting text from new images...")
-new_images = combined_df[~combined_df['image_url'].isin(cache.keys())]
+new_images = your_df[~your_df['image_url'].isin(cache.keys())]
 new_images = process_new_images(new_images)
 
-# Update the combined DataFrame with extracted text
-combined_df.update(new_images)
+# Update the DataFrame with extracted text
+your_df.update(new_images)
 
-# Save the updated dataset to Hugging Face Hub
-save_to_hub({"data_with_extracted_text": combined_df.to_dict(), CACHE_KEY: cache}, DATASET_NAME)
+# Save the cache
+with open(CACHE_FILE, "wb") as f:
+    pickle.dump(cache, f)
 
 logging.info("Incremental processing complete!")
 
@@ -245,7 +176,7 @@ def analyze_image(image_url):
         return None
 
 logging.info("Analyzing image content...")
-combined_df['image_analysis'] = combined_df['image_url'].apply(analyze_image)
+your_df['image_analysis'] = your_df['image_url'].apply(analyze_image)
 
 # Define scoring criteria for posts
 def rate_post(row):
@@ -264,11 +195,11 @@ def rate_post(row):
     return score
 
 logging.info("Rating posts...")
-combined_df['post_score'] = combined_df.apply(rate_post, axis=1)
+your_df['post_score'] = your_df.apply(rate_post, axis=1)
 
 # Log the results
 logging.info("Post Ratings:")
-print(combined_df[['post_id', 'post_score']].head())
+print(your_df[['post_id', 'post_score']].head())
 
 # Train models using all features
 logging.info("Training models using all features...")
@@ -280,11 +211,11 @@ features = [
 
 # Viral Potential Prediction
 logging.info("Training viral potential prediction model...")
-combined_viral_threshold = combined_df['engagement_rate'].quantile(0.9)
-combined_df['viral'] = combined_df['engagement_rate'].apply(lambda x: 1 if x >= combined_viral_threshold else 0)
+viral_threshold = your_df['engagement_rate'].quantile(0.9)
+your_df['viral'] = your_df['engagement_rate'].apply(lambda x: 1 if x >= viral_threshold else 0)
 
-X = combined_df[features]
-y = combined_df['viral']
+X = your_df[features]
+y = your_df['viral']
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 
 viral_model = RandomForestClassifier(random_state=42)
@@ -295,8 +226,8 @@ logging.info(f"Viral Potential Model Accuracy: {accuracy:.4f}")
 
 # Engagement Rate Prediction
 logging.info("Training engagement rate prediction model...")
-X = combined_df[features]
-y = combined_df['engagement_rate']
+X = your_df[features]
+y = your_df['engagement_rate']
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 
 engagement_model = XGBRegressor(random_state=42)
@@ -307,11 +238,11 @@ logging.info(f"Engagement Rate Prediction Model - MAE: {mae:.4f}")
 
 # Promotion Strategy
 logging.info("Training promotion prediction model...")
-promotion_threshold = combined_df['engagement_rate'].quantile(0.8)
-combined_df['promote'] = combined_df['engagement_rate'].apply(lambda x: 1 if x >= promotion_threshold else 0)
+promotion_threshold = your_df['engagement_rate'].quantile(0.8)
+your_df['promote'] = your_df['engagement_rate'].apply(lambda x: 1 if x >= promotion_threshold else 0)
 
-X = combined_df[features]
-y = combined_df['promote']
+X = your_df[features]
+y = your_df['promote']
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 
 promotion_model = LogisticRegression(random_state=42)