Fred808 commited on
Commit
4e07d03
·
verified ·
1 Parent(s): 6df78c3

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +67 -112
app.py CHANGED
@@ -19,7 +19,10 @@ from textblob import TextBlob # For sentiment analysis
19
  from imblearn.over_sampling import SMOTE # For handling imbalanced data
20
  import logging
21
  import matplotlib.pyplot as plt
 
22
  from statsmodels.tsa.stattools import adfuller # For stationarity check
 
 
23
 
24
  # Set up logging
25
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -122,119 +125,71 @@ combined_df = pd.concat([solved_df, competitor_df], ignore_index=True)
122
  # Log the column names of the combined DataFrame
123
  logging.info(f"Combined DataFrame Columns: {combined_df.columns.tolist()}")
124
 
125
- # Enhanced niche trend analysis using combined data
126
- logging.info("Analyzing enhanced niche trends using combined data...")
127
- combined_niche_trends = combined_df.groupby('content_type')['sentiment'].mean().sort_values(ascending=False)
128
- logging.info("Top Performing Content Types by Sentiment (Combined Data):")
129
- print(combined_niche_trends)
130
-
131
- # Encode categorical columns if they exist
132
- if 'content_type' in combined_df.columns and 'media_type' in combined_df.columns:
133
- logging.info("Encoding categorical columns...")
134
- label_encoder = LabelEncoder()
135
- combined_df['content_type_encoded'] = label_encoder.fit_transform(combined_df['content_type'])
136
- combined_df['media_type_encoded'] = label_encoder.fit_transform(combined_df['media_type'])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  else:
138
- logging.warning("'content_type' or 'media_type' columns not found. Skipping encoding.")
139
 
140
- # Update the features list
141
- if 'content_type_encoded' in combined_df.columns and 'media_type_encoded' in combined_df.columns:
142
- features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded']
143
- else:
144
- logging.warning("Using reduced feature set because encoded columns are missing.")
145
- features = ['caption_length', 'hashtag_count', 'sentiment']
146
-
147
- # Log the features being used
148
- logging.info(f"Features for model training: {features}")
149
-
150
- # Enhanced viral potential prediction using combined data
151
- logging.info("Training enhanced viral potential prediction model using combined data...")
152
- combined_viral_threshold = combined_df['engagement_rate'].quantile(0.9)
153
- combined_df['viral'] = combined_df['engagement_rate'].apply(lambda x: 1 if x >= combined_viral_threshold else 0)
154
-
155
- # Split data into training and testing sets
156
- X = combined_df[features]
157
- y = combined_df['viral']
158
-
159
- X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
160
-
161
- # Train a Random Forest Classifier
162
- viral_model = RandomForestClassifier(random_state=42)
163
- viral_model.fit(X_train, y_train)
164
-
165
- # Evaluate the model
166
- y_pred = viral_model.predict(X_test)
167
- accuracy = accuracy_score(y_test, y_pred)
168
- logging.info(f"Enhanced Viral Potential Model Accuracy: {accuracy:.4f}")
169
-
170
- # Feature importance
171
- importance = viral_model.feature_importances_
172
- for feature, score in zip(features, importance):
173
- logging.info(f"Feature Importance - {feature}: {score:.4f}")
174
-
175
- # Check for invalid values in 'engagement_rate'
176
- logging.info("Checking for invalid values in 'engagement_rate'...")
177
-
178
- # Check for NaN values
179
- if combined_df['engagement_rate'].isnull().any():
180
- logging.warning("NaN values found in 'engagement_rate'. Filling with 0.")
181
- combined_df['engagement_rate'].fillna(0, inplace=True)
182
-
183
- # Check for infinity or extremely large values
184
- if np.isinf(combined_df['engagement_rate']).any() or (combined_df['engagement_rate'] > 1e6).any():
185
- logging.warning("Infinity or extremely large values found in 'engagement_rate'. Clipping values.")
186
- combined_df['engagement_rate'] = np.clip(combined_df['engagement_rate'], -1e6, 1e6) # Clip to reasonable range
187
-
188
- # Log the updated 'engagement_rate' column
189
- logging.info("Updated 'engagement_rate' column:")
190
- print(combined_df['engagement_rate'].describe())
191
-
192
- # Enhanced engagement rate predictions using combined data
193
- logging.info("Training enhanced engagement rate prediction model using combined data...")
194
- X = combined_df[features]
195
- y = combined_df['engagement_rate']
196
-
197
- # Split data into training and testing sets
198
- X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
199
-
200
- # Train an XGBoost Regressor
201
- engagement_model = XGBRegressor(random_state=42)
202
- engagement_model.fit(X_train, y_train)
203
-
204
- # Evaluate the model
205
- y_pred = engagement_model.predict(X_test)
206
- mae = mean_absolute_error(y_test, y_pred)
207
- logging.info(f"Enhanced Engagement Rate Prediction Model - MAE: {mae:.4f}")
208
-
209
- # Feature importance
210
- importance = engagement_model.feature_importances_
211
- for feature, score in zip(features, importance):
212
- logging.info(f"Feature Importance - {feature}: {score:.4f}")
213
-
214
- # Enhanced promotion strategy using combined data
215
- logging.info("Training enhanced promotion prediction model using combined data...")
216
- promotion_threshold = combined_df['engagement_rate'].quantile(0.8)
217
- combined_df['promote'] = combined_df['engagement_rate'].apply(lambda x: 1 if x >= promotion_threshold else 0)
218
-
219
- # Features for promotion prediction
220
- X = combined_df[features]
221
- y = combined_df['promote']
222
-
223
- # Split data into training and testing sets
224
- X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
225
-
226
- # Train a Logistic Regression Model
227
- promotion_model = LogisticRegression(random_state=42)
228
- promotion_model.fit(X_train, y_train)
229
-
230
- # Evaluate the model
231
- y_pred = promotion_model.predict(X_test)
232
- accuracy = accuracy_score(y_test, y_pred)
233
- logging.info(f"Enhanced Promotion Prediction Model Accuracy: {accuracy:.4f}")
234
-
235
- # Analyze content type impact
236
- content_type_impact = combined_df.groupby('content_type')['promote'].mean().sort_values(ascending=False)
237
- logging.info("Enhanced Content Type Impact on Promotion:")
238
- print(content_type_impact)
239
 
240
  logging.info("Enhanced analysis complete!")
 
19
  from imblearn.over_sampling import SMOTE # For handling imbalanced data
20
  import logging
21
  import matplotlib.pyplot as plt
22
+ import seaborn as sns
23
  from statsmodels.tsa.stattools import adfuller # For stationarity check
24
+ from collections import Counter
25
+ import re
26
 
27
  # Set up logging
28
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
125
  # Log the column names of the combined DataFrame
126
  logging.info(f"Combined DataFrame Columns: {combined_df.columns.tolist()}")
127
 
128
+ # Engagement Heatmap by Time of Day
129
+ logging.info("Creating engagement heatmap by time of day...")
130
+ engagement_df['hour'] = engagement_df['posting_time'].dt.hour
131
+ engagement_by_hour = engagement_df.groupby('hour')['engagement_rate'].mean().reset_index()
132
+
133
+ plt.figure(figsize=(10, 6))
134
+ sns.heatmap(engagement_by_hour.pivot_table(index='hour', values='engagement_rate'), annot=True, cmap='YlGnBu')
135
+ plt.title('Engagement Heatmap by Time of Day')
136
+ plt.xlabel('Engagement Rate')
137
+ plt.ylabel('Hour of Day')
138
+ plt.show()
139
+
140
+ # Engagement Breakdown by Content Type
141
+ logging.info("Analyzing engagement by content type...")
142
+ engagement_by_content_type = combined_df.groupby('content_type')['engagement_rate'].mean().sort_values(ascending=False)
143
+ logging.info("Engagement by Content Type:")
144
+ print(engagement_by_content_type)
145
+
146
+ # Trending Hashtags
147
+ logging.info("Analyzing trending hashtags...")
148
+ all_hashtags = combined_df['hashtags'].explode().value_counts().head(10)
149
+ logging.info("Top 10 Trending Hashtags:")
150
+ print(all_hashtags)
151
+
152
+ # Trending Keywords in Captions
153
+ logging.info("Analyzing trending keywords in captions...")
154
+ words = combined_df['caption'].apply(lambda x: re.findall(r'\b\w+\b', x.lower())).explode()
155
+ word_counts = Counter(words).most_common(10)
156
+ logging.info("Top 10 Trending Keywords in Captions:")
157
+ print(word_counts)
158
+
159
+ # Competitor Engagement Metrics
160
+ logging.info("Analyzing competitor engagement metrics...")
161
+ competitor_engagement = competitor_df.groupby('post_id').agg({
162
+ 'likes': 'sum',
163
+ 'comments': 'sum',
164
+ 'shares': 'sum',
165
+ 'engagement_rate': 'mean'
166
+ }).reset_index()
167
+
168
+ logging.info("Competitor Engagement Metrics:")
169
+ print(competitor_engagement.describe())
170
+
171
+ # Follower Growth Trends (if available)
172
+ if 'followers' in competitor_df.columns:
173
+ logging.info("Analyzing competitor follower growth trends...")
174
+ competitor_follower_growth = competitor_df.groupby('posting_time')['followers'].sum().reset_index()
175
+ plt.figure(figsize=(10, 6))
176
+ plt.plot(competitor_follower_growth['posting_time'], competitor_follower_growth['followers'], label='Competitor')
177
+ plt.title('Competitor Follower Growth Over Time')
178
+ plt.xlabel('Time')
179
+ plt.ylabel('Followers')
180
+ plt.legend()
181
+ plt.show()
182
  else:
183
+ logging.warning("Follower data not found in competitor data. Skipping follower growth analysis.")
184
 
185
+ # Competitor Hashtags and Posting Schedules
186
+ logging.info("Analyzing competitor hashtags and posting schedules...")
187
+ competitor_hashtags = competitor_df['hashtags'].explode().value_counts().head(10)
188
+ logging.info("Top 10 Competitor Hashtags:")
189
+ print(competitor_hashtags)
190
+
191
+ competitor_posting_schedule = competitor_df['posting_time'].dt.hour.value_counts().sort_index()
192
+ logging.info("Competitor Posting Schedule (by hour):")
193
+ print(competitor_posting_schedule)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
194
 
195
  logging.info("Enhanced analysis complete!")