Update app.py
Browse files
app.py
CHANGED
|
@@ -19,7 +19,10 @@ from textblob import TextBlob # For sentiment analysis
|
|
| 19 |
from imblearn.over_sampling import SMOTE # For handling imbalanced data
|
| 20 |
import logging
|
| 21 |
import matplotlib.pyplot as plt
|
|
|
|
| 22 |
from statsmodels.tsa.stattools import adfuller # For stationarity check
|
|
|
|
|
|
|
| 23 |
|
| 24 |
# Set up logging
|
| 25 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
@@ -122,119 +125,71 @@ combined_df = pd.concat([solved_df, competitor_df], ignore_index=True)
|
|
| 122 |
# Log the column names of the combined DataFrame
|
| 123 |
logging.info(f"Combined DataFrame Columns: {combined_df.columns.tolist()}")
|
| 124 |
|
| 125 |
-
#
|
| 126 |
-
logging.info("
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
else:
|
| 138 |
-
logging.warning("
|
| 139 |
|
| 140 |
-
#
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
# Enhanced viral potential prediction using combined data
|
| 151 |
-
logging.info("Training enhanced viral potential prediction model using combined data...")
|
| 152 |
-
combined_viral_threshold = combined_df['engagement_rate'].quantile(0.9)
|
| 153 |
-
combined_df['viral'] = combined_df['engagement_rate'].apply(lambda x: 1 if x >= combined_viral_threshold else 0)
|
| 154 |
-
|
| 155 |
-
# Split data into training and testing sets
|
| 156 |
-
X = combined_df[features]
|
| 157 |
-
y = combined_df['viral']
|
| 158 |
-
|
| 159 |
-
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
|
| 160 |
-
|
| 161 |
-
# Train a Random Forest Classifier
|
| 162 |
-
viral_model = RandomForestClassifier(random_state=42)
|
| 163 |
-
viral_model.fit(X_train, y_train)
|
| 164 |
-
|
| 165 |
-
# Evaluate the model
|
| 166 |
-
y_pred = viral_model.predict(X_test)
|
| 167 |
-
accuracy = accuracy_score(y_test, y_pred)
|
| 168 |
-
logging.info(f"Enhanced Viral Potential Model Accuracy: {accuracy:.4f}")
|
| 169 |
-
|
| 170 |
-
# Feature importance
|
| 171 |
-
importance = viral_model.feature_importances_
|
| 172 |
-
for feature, score in zip(features, importance):
|
| 173 |
-
logging.info(f"Feature Importance - {feature}: {score:.4f}")
|
| 174 |
-
|
| 175 |
-
# Check for invalid values in 'engagement_rate'
|
| 176 |
-
logging.info("Checking for invalid values in 'engagement_rate'...")
|
| 177 |
-
|
| 178 |
-
# Check for NaN values
|
| 179 |
-
if combined_df['engagement_rate'].isnull().any():
|
| 180 |
-
logging.warning("NaN values found in 'engagement_rate'. Filling with 0.")
|
| 181 |
-
combined_df['engagement_rate'].fillna(0, inplace=True)
|
| 182 |
-
|
| 183 |
-
# Check for infinity or extremely large values
|
| 184 |
-
if np.isinf(combined_df['engagement_rate']).any() or (combined_df['engagement_rate'] > 1e6).any():
|
| 185 |
-
logging.warning("Infinity or extremely large values found in 'engagement_rate'. Clipping values.")
|
| 186 |
-
combined_df['engagement_rate'] = np.clip(combined_df['engagement_rate'], -1e6, 1e6) # Clip to reasonable range
|
| 187 |
-
|
| 188 |
-
# Log the updated 'engagement_rate' column
|
| 189 |
-
logging.info("Updated 'engagement_rate' column:")
|
| 190 |
-
print(combined_df['engagement_rate'].describe())
|
| 191 |
-
|
| 192 |
-
# Enhanced engagement rate predictions using combined data
|
| 193 |
-
logging.info("Training enhanced engagement rate prediction model using combined data...")
|
| 194 |
-
X = combined_df[features]
|
| 195 |
-
y = combined_df['engagement_rate']
|
| 196 |
-
|
| 197 |
-
# Split data into training and testing sets
|
| 198 |
-
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
|
| 199 |
-
|
| 200 |
-
# Train an XGBoost Regressor
|
| 201 |
-
engagement_model = XGBRegressor(random_state=42)
|
| 202 |
-
engagement_model.fit(X_train, y_train)
|
| 203 |
-
|
| 204 |
-
# Evaluate the model
|
| 205 |
-
y_pred = engagement_model.predict(X_test)
|
| 206 |
-
mae = mean_absolute_error(y_test, y_pred)
|
| 207 |
-
logging.info(f"Enhanced Engagement Rate Prediction Model - MAE: {mae:.4f}")
|
| 208 |
-
|
| 209 |
-
# Feature importance
|
| 210 |
-
importance = engagement_model.feature_importances_
|
| 211 |
-
for feature, score in zip(features, importance):
|
| 212 |
-
logging.info(f"Feature Importance - {feature}: {score:.4f}")
|
| 213 |
-
|
| 214 |
-
# Enhanced promotion strategy using combined data
|
| 215 |
-
logging.info("Training enhanced promotion prediction model using combined data...")
|
| 216 |
-
promotion_threshold = combined_df['engagement_rate'].quantile(0.8)
|
| 217 |
-
combined_df['promote'] = combined_df['engagement_rate'].apply(lambda x: 1 if x >= promotion_threshold else 0)
|
| 218 |
-
|
| 219 |
-
# Features for promotion prediction
|
| 220 |
-
X = combined_df[features]
|
| 221 |
-
y = combined_df['promote']
|
| 222 |
-
|
| 223 |
-
# Split data into training and testing sets
|
| 224 |
-
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
|
| 225 |
-
|
| 226 |
-
# Train a Logistic Regression Model
|
| 227 |
-
promotion_model = LogisticRegression(random_state=42)
|
| 228 |
-
promotion_model.fit(X_train, y_train)
|
| 229 |
-
|
| 230 |
-
# Evaluate the model
|
| 231 |
-
y_pred = promotion_model.predict(X_test)
|
| 232 |
-
accuracy = accuracy_score(y_test, y_pred)
|
| 233 |
-
logging.info(f"Enhanced Promotion Prediction Model Accuracy: {accuracy:.4f}")
|
| 234 |
-
|
| 235 |
-
# Analyze content type impact
|
| 236 |
-
content_type_impact = combined_df.groupby('content_type')['promote'].mean().sort_values(ascending=False)
|
| 237 |
-
logging.info("Enhanced Content Type Impact on Promotion:")
|
| 238 |
-
print(content_type_impact)
|
| 239 |
|
| 240 |
logging.info("Enhanced analysis complete!")
|
|
|
|
| 19 |
from imblearn.over_sampling import SMOTE # For handling imbalanced data
|
| 20 |
import logging
|
| 21 |
import matplotlib.pyplot as plt
|
| 22 |
+
import seaborn as sns
|
| 23 |
from statsmodels.tsa.stattools import adfuller # For stationarity check
|
| 24 |
+
from collections import Counter
|
| 25 |
+
import re
|
| 26 |
|
| 27 |
# Set up logging
|
| 28 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
|
|
| 125 |
# Log the column names of the combined DataFrame
|
| 126 |
logging.info(f"Combined DataFrame Columns: {combined_df.columns.tolist()}")
|
| 127 |
|
| 128 |
+
# Engagement Heatmap by Time of Day
|
| 129 |
+
logging.info("Creating engagement heatmap by time of day...")
|
| 130 |
+
engagement_df['hour'] = engagement_df['posting_time'].dt.hour
|
| 131 |
+
engagement_by_hour = engagement_df.groupby('hour')['engagement_rate'].mean().reset_index()
|
| 132 |
+
|
| 133 |
+
plt.figure(figsize=(10, 6))
|
| 134 |
+
sns.heatmap(engagement_by_hour.pivot_table(index='hour', values='engagement_rate'), annot=True, cmap='YlGnBu')
|
| 135 |
+
plt.title('Engagement Heatmap by Time of Day')
|
| 136 |
+
plt.xlabel('Engagement Rate')
|
| 137 |
+
plt.ylabel('Hour of Day')
|
| 138 |
+
plt.show()
|
| 139 |
+
|
| 140 |
+
# Engagement Breakdown by Content Type
|
| 141 |
+
logging.info("Analyzing engagement by content type...")
|
| 142 |
+
engagement_by_content_type = combined_df.groupby('content_type')['engagement_rate'].mean().sort_values(ascending=False)
|
| 143 |
+
logging.info("Engagement by Content Type:")
|
| 144 |
+
print(engagement_by_content_type)
|
| 145 |
+
|
| 146 |
+
# Trending Hashtags
|
| 147 |
+
logging.info("Analyzing trending hashtags...")
|
| 148 |
+
all_hashtags = combined_df['hashtags'].explode().value_counts().head(10)
|
| 149 |
+
logging.info("Top 10 Trending Hashtags:")
|
| 150 |
+
print(all_hashtags)
|
| 151 |
+
|
| 152 |
+
# Trending Keywords in Captions
|
| 153 |
+
logging.info("Analyzing trending keywords in captions...")
|
| 154 |
+
words = combined_df['caption'].apply(lambda x: re.findall(r'\b\w+\b', x.lower())).explode()
|
| 155 |
+
word_counts = Counter(words).most_common(10)
|
| 156 |
+
logging.info("Top 10 Trending Keywords in Captions:")
|
| 157 |
+
print(word_counts)
|
| 158 |
+
|
| 159 |
+
# Competitor Engagement Metrics
|
| 160 |
+
logging.info("Analyzing competitor engagement metrics...")
|
| 161 |
+
competitor_engagement = competitor_df.groupby('post_id').agg({
|
| 162 |
+
'likes': 'sum',
|
| 163 |
+
'comments': 'sum',
|
| 164 |
+
'shares': 'sum',
|
| 165 |
+
'engagement_rate': 'mean'
|
| 166 |
+
}).reset_index()
|
| 167 |
+
|
| 168 |
+
logging.info("Competitor Engagement Metrics:")
|
| 169 |
+
print(competitor_engagement.describe())
|
| 170 |
+
|
| 171 |
+
# Follower Growth Trends (if available)
|
| 172 |
+
if 'followers' in competitor_df.columns:
|
| 173 |
+
logging.info("Analyzing competitor follower growth trends...")
|
| 174 |
+
competitor_follower_growth = competitor_df.groupby('posting_time')['followers'].sum().reset_index()
|
| 175 |
+
plt.figure(figsize=(10, 6))
|
| 176 |
+
plt.plot(competitor_follower_growth['posting_time'], competitor_follower_growth['followers'], label='Competitor')
|
| 177 |
+
plt.title('Competitor Follower Growth Over Time')
|
| 178 |
+
plt.xlabel('Time')
|
| 179 |
+
plt.ylabel('Followers')
|
| 180 |
+
plt.legend()
|
| 181 |
+
plt.show()
|
| 182 |
else:
|
| 183 |
+
logging.warning("Follower data not found in competitor data. Skipping follower growth analysis.")
|
| 184 |
|
| 185 |
+
# Competitor Hashtags and Posting Schedules
|
| 186 |
+
logging.info("Analyzing competitor hashtags and posting schedules...")
|
| 187 |
+
competitor_hashtags = competitor_df['hashtags'].explode().value_counts().head(10)
|
| 188 |
+
logging.info("Top 10 Competitor Hashtags:")
|
| 189 |
+
print(competitor_hashtags)
|
| 190 |
+
|
| 191 |
+
competitor_posting_schedule = competitor_df['posting_time'].dt.hour.value_counts().sort_index()
|
| 192 |
+
logging.info("Competitor Posting Schedule (by hour):")
|
| 193 |
+
print(competitor_posting_schedule)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
|
| 195 |
logging.info("Enhanced analysis complete!")
|