JaySMTA commited on
Commit
9e4317b
·
verified ·
1 Parent(s): c2a59f0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +57 -63
app.py CHANGED
@@ -92,35 +92,7 @@ def scrape_facebook_profile(url):
92
  logging.info(f"Facebook API response: {json.dumps(result, indent=2)}")
93
  if 'error' not in result:
94
  logging.info(f"Successfully scraped Facebook profile: {url}")
95
- profile_data = {
96
- "Name": result.get('name', ''),
97
- "Profile Link": url,
98
- "Bio": result.get('bio', ''),
99
- "About Info": result.get('about', ''),
100
- "Websites": result.get('websites', []),
101
- "Social Media Links": result.get('social_media_links', []),
102
- "Emails": result.get('emails', []),
103
- "Current Location": result.get('current_location', ''),
104
- "Age": result.get('age', ''),
105
- "Gender": result.get('gender', ''),
106
- "Profession": result.get('profession', ''),
107
- "Friends Count": result.get('friends_count', ''),
108
- "Followers Count": result.get('followers_count', ''),
109
- "Following Count": result.get('following_count', ''),
110
- }
111
-
112
- # Fetch the latest 45 posts
113
- posts = result.get('posts', [])[:45]
114
- profile_data['Latest Posts'] = [{
115
- "Content": post.get('content', ''),
116
- "Has Image": 'Yes' if post.get('image_url') else 'No',
117
- "Likes Count": post.get('likes_count', ''),
118
- "Comments Count": post.get('comments_count', ''),
119
- "Shares Count": post.get('shares_count', ''),
120
- "Date": post.get('date', '')
121
- } for post in posts]
122
-
123
- return profile_data
124
  else:
125
  logging.error(f"API error for Facebook profile {url}: {result.get('error')}")
126
  return None
@@ -218,43 +190,65 @@ def create_interface(name, scrape_function):
218
  result = process_csv(csv_file, scrape_function)
219
  if result.startswith("Error"):
220
  return None
221
- try:
222
- data = json.loads(result)
223
- if not isinstance(data, list):
224
- logging.error(f"Unexpected data format: {type(data)}")
225
- return None
226
- if not data:
227
- logging.error("No data to write to CSV")
228
- return None
229
- output_file = f"{name.lower()}_scraped_data.csv"
230
- with open(output_file, 'w', newline='', encoding='utf-8') as f:
231
- fieldnames = list(data[0].keys())
232
- if 'Latest Posts' in fieldnames:
233
- fieldnames.remove('Latest Posts')
234
- for i in range(45):
235
- fieldnames.extend([f"Post {i+1} Content", f"Post {i+1} Has Image",
236
- f"Post {i+1} Likes Count", f"Post {i+1} Comments Count",
237
- f"Post {i+1} Shares Count", f"Post {i+1} Date"])
238
  writer = csv.DictWriter(f, fieldnames=fieldnames)
239
  writer.writeheader()
 
240
  for row in data:
241
- flat_row = {k: v for k, v in row.items() if k != 'Latest Posts'}
242
- if 'Latest Posts' in row:
243
- for i, post in enumerate(row['Latest Posts']):
244
- flat_row[f"Post {i+1} Content"] = post['Content']
245
- flat_row[f"Post {i+1} Has Image"] = post['Has Image']
246
- flat_row[f"Post {i+1} Likes Count"] = post['Likes Count']
247
- flat_row[f"Post {i+1} Comments Count"] = post['Comments Count']
248
- flat_row[f"Post {i+1} Shares Count"] = post['Shares Count']
249
- flat_row[f"Post {i+1} Date"] = post['Date']
250
- writer.writerow(flat_row)
251
- return output_file
252
- except json.JSONDecodeError:
253
- logging.error(f"Failed to parse JSON: {result}")
254
- return None
255
- except Exception as e:
256
- logging.error(f"Error in scrape_and_download: {str(e)}")
257
- return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258
 
259
  preview_button.click(lambda x: preview_csv(x, scrape_function), inputs=input_file, outputs=preview)
260
  clear_preview_button.click(clear_preview, inputs=None, outputs=preview)
 
92
  logging.info(f"Facebook API response: {json.dumps(result, indent=2)}")
93
  if 'error' not in result:
94
  logging.info(f"Successfully scraped Facebook profile: {url}")
95
+ return result
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  else:
97
  logging.error(f"API error for Facebook profile {url}: {result.get('error')}")
98
  return None
 
190
  result = process_csv(csv_file, scrape_function)
191
  if result.startswith("Error"):
192
  return None
193
+ data = json.loads(result)
194
+ output_file = f"{name.lower()}_scraped_data.csv"
195
+
196
+ def clean_text(text):
197
+ return ' '.join(str(text).replace('\n', ' ').split())
198
+
199
+ with open(output_file, 'w', newline='', encoding='utf-8') as f:
200
+ if data:
201
+ fieldnames = ['firstName', 'lastName', 'fullName', 'headline', 'location', 'connections', 'about']
202
+ for i in range(5):
203
+ fieldnames.extend([f'experience_{i+1}_title', f'experience_{i+1}_company', f'experience_{i+1}_duration'])
204
+ for i in range(3):
205
+ fieldnames.extend([f'education_{i+1}_school', f'education_{i+1}_degree', f'education_{i+1}_date'])
206
+ fieldnames.extend(['languages', 'skills'])
207
+
 
 
208
  writer = csv.DictWriter(f, fieldnames=fieldnames)
209
  writer.writeheader()
210
+
211
  for row in data:
212
+ clean_row = {
213
+ 'firstName': clean_text(row.get('firstName', '')),
214
+ 'lastName': clean_text(row.get('lastName', '')),
215
+ 'fullName': clean_text(row.get('fullName', '')),
216
+ 'headline': clean_text(row.get('headline', '')),
217
+ 'location': clean_text(row.get('addressWithCountry', '')),
218
+ 'connections': row.get('connections', ''),
219
+ 'about': clean_text(row.get('about', ''))
220
+ }
221
+
222
+ experiences = row.get('experiences', [])
223
+ for i in range(5):
224
+ if i < len(experiences):
225
+ exp = experiences[i]
226
+ clean_row[f'experience_{i+1}_title'] = clean_text(exp.get('title', ''))
227
+ clean_row[f'experience_{i+1}_company'] = clean_text(exp.get('subtitle', ''))
228
+ clean_row[f'experience_{i+1}_duration'] = clean_text(exp.get('caption', ''))
229
+ else:
230
+ clean_row[f'experience_{i+1}_title'] = ''
231
+ clean_row[f'experience_{i+1}_company'] = ''
232
+ clean_row[f'experience_{i+1}_duration'] = ''
233
+
234
+ educations = row.get('educations', [])
235
+ for i in range(3):
236
+ if i < len(educations):
237
+ edu = educations[i]
238
+ clean_row[f'education_{i+1}_school'] = clean_text(edu.get('title', ''))
239
+ clean_row[f'education_{i+1}_degree'] = clean_text(edu.get('subtitle', ''))
240
+ clean_row[f'education_{i+1}_date'] = clean_text(edu.get('caption', ''))
241
+ else:
242
+ clean_row[f'education_{i+1}_school'] = ''
243
+ clean_row[f'education_{i+1}_degree'] = ''
244
+ clean_row[f'education_{i+1}_date'] = ''
245
+
246
+ clean_row['languages'] = ', '.join([lang.get('title', '') for lang in row.get('languages', [])])
247
+ clean_row['skills'] = ', '.join([skill.get('title', '') for skill in row.get('skills', [])[:10]])
248
+
249
+ writer.writerow(clean_row)
250
+
251
+ return output_file
252
 
253
  preview_button.click(lambda x: preview_csv(x, scrape_function), inputs=input_file, outputs=preview)
254
  clear_preview_button.click(clear_preview, inputs=None, outputs=preview)