Update app.py
Browse files
app.py
CHANGED
|
@@ -92,35 +92,7 @@ def scrape_facebook_profile(url):
|
|
| 92 |
logging.info(f"Facebook API response: {json.dumps(result, indent=2)}")
|
| 93 |
if 'error' not in result:
|
| 94 |
logging.info(f"Successfully scraped Facebook profile: {url}")
|
| 95 |
-
|
| 96 |
-
"Name": result.get('name', ''),
|
| 97 |
-
"Profile Link": url,
|
| 98 |
-
"Bio": result.get('bio', ''),
|
| 99 |
-
"About Info": result.get('about', ''),
|
| 100 |
-
"Websites": result.get('websites', []),
|
| 101 |
-
"Social Media Links": result.get('social_media_links', []),
|
| 102 |
-
"Emails": result.get('emails', []),
|
| 103 |
-
"Current Location": result.get('current_location', ''),
|
| 104 |
-
"Age": result.get('age', ''),
|
| 105 |
-
"Gender": result.get('gender', ''),
|
| 106 |
-
"Profession": result.get('profession', ''),
|
| 107 |
-
"Friends Count": result.get('friends_count', ''),
|
| 108 |
-
"Followers Count": result.get('followers_count', ''),
|
| 109 |
-
"Following Count": result.get('following_count', ''),
|
| 110 |
-
}
|
| 111 |
-
|
| 112 |
-
# Fetch the latest 45 posts
|
| 113 |
-
posts = result.get('posts', [])[:45]
|
| 114 |
-
profile_data['Latest Posts'] = [{
|
| 115 |
-
"Content": post.get('content', ''),
|
| 116 |
-
"Has Image": 'Yes' if post.get('image_url') else 'No',
|
| 117 |
-
"Likes Count": post.get('likes_count', ''),
|
| 118 |
-
"Comments Count": post.get('comments_count', ''),
|
| 119 |
-
"Shares Count": post.get('shares_count', ''),
|
| 120 |
-
"Date": post.get('date', '')
|
| 121 |
-
} for post in posts]
|
| 122 |
-
|
| 123 |
-
return profile_data
|
| 124 |
else:
|
| 125 |
logging.error(f"API error for Facebook profile {url}: {result.get('error')}")
|
| 126 |
return None
|
|
@@ -218,43 +190,65 @@ def create_interface(name, scrape_function):
|
|
| 218 |
result = process_csv(csv_file, scrape_function)
|
| 219 |
if result.startswith("Error"):
|
| 220 |
return None
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
fieldnames.
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
f"Post {i+1} Likes Count", f"Post {i+1} Comments Count",
|
| 237 |
-
f"Post {i+1} Shares Count", f"Post {i+1} Date"])
|
| 238 |
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
| 239 |
writer.writeheader()
|
|
|
|
| 240 |
for row in data:
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
|
| 259 |
preview_button.click(lambda x: preview_csv(x, scrape_function), inputs=input_file, outputs=preview)
|
| 260 |
clear_preview_button.click(clear_preview, inputs=None, outputs=preview)
|
|
|
|
| 92 |
logging.info(f"Facebook API response: {json.dumps(result, indent=2)}")
|
| 93 |
if 'error' not in result:
|
| 94 |
logging.info(f"Successfully scraped Facebook profile: {url}")
|
| 95 |
+
return result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
else:
|
| 97 |
logging.error(f"API error for Facebook profile {url}: {result.get('error')}")
|
| 98 |
return None
|
|
|
|
| 190 |
result = process_csv(csv_file, scrape_function)
|
| 191 |
if result.startswith("Error"):
|
| 192 |
return None
|
| 193 |
+
data = json.loads(result)
|
| 194 |
+
output_file = f"{name.lower()}_scraped_data.csv"
|
| 195 |
+
|
| 196 |
+
def clean_text(text):
|
| 197 |
+
return ' '.join(str(text).replace('\n', ' ').split())
|
| 198 |
+
|
| 199 |
+
with open(output_file, 'w', newline='', encoding='utf-8') as f:
|
| 200 |
+
if data:
|
| 201 |
+
fieldnames = ['firstName', 'lastName', 'fullName', 'headline', 'location', 'connections', 'about']
|
| 202 |
+
for i in range(5):
|
| 203 |
+
fieldnames.extend([f'experience_{i+1}_title', f'experience_{i+1}_company', f'experience_{i+1}_duration'])
|
| 204 |
+
for i in range(3):
|
| 205 |
+
fieldnames.extend([f'education_{i+1}_school', f'education_{i+1}_degree', f'education_{i+1}_date'])
|
| 206 |
+
fieldnames.extend(['languages', 'skills'])
|
| 207 |
+
|
|
|
|
|
|
|
| 208 |
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
| 209 |
writer.writeheader()
|
| 210 |
+
|
| 211 |
for row in data:
|
| 212 |
+
clean_row = {
|
| 213 |
+
'firstName': clean_text(row.get('firstName', '')),
|
| 214 |
+
'lastName': clean_text(row.get('lastName', '')),
|
| 215 |
+
'fullName': clean_text(row.get('fullName', '')),
|
| 216 |
+
'headline': clean_text(row.get('headline', '')),
|
| 217 |
+
'location': clean_text(row.get('addressWithCountry', '')),
|
| 218 |
+
'connections': row.get('connections', ''),
|
| 219 |
+
'about': clean_text(row.get('about', ''))
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
experiences = row.get('experiences', [])
|
| 223 |
+
for i in range(5):
|
| 224 |
+
if i < len(experiences):
|
| 225 |
+
exp = experiences[i]
|
| 226 |
+
clean_row[f'experience_{i+1}_title'] = clean_text(exp.get('title', ''))
|
| 227 |
+
clean_row[f'experience_{i+1}_company'] = clean_text(exp.get('subtitle', ''))
|
| 228 |
+
clean_row[f'experience_{i+1}_duration'] = clean_text(exp.get('caption', ''))
|
| 229 |
+
else:
|
| 230 |
+
clean_row[f'experience_{i+1}_title'] = ''
|
| 231 |
+
clean_row[f'experience_{i+1}_company'] = ''
|
| 232 |
+
clean_row[f'experience_{i+1}_duration'] = ''
|
| 233 |
+
|
| 234 |
+
educations = row.get('educations', [])
|
| 235 |
+
for i in range(3):
|
| 236 |
+
if i < len(educations):
|
| 237 |
+
edu = educations[i]
|
| 238 |
+
clean_row[f'education_{i+1}_school'] = clean_text(edu.get('title', ''))
|
| 239 |
+
clean_row[f'education_{i+1}_degree'] = clean_text(edu.get('subtitle', ''))
|
| 240 |
+
clean_row[f'education_{i+1}_date'] = clean_text(edu.get('caption', ''))
|
| 241 |
+
else:
|
| 242 |
+
clean_row[f'education_{i+1}_school'] = ''
|
| 243 |
+
clean_row[f'education_{i+1}_degree'] = ''
|
| 244 |
+
clean_row[f'education_{i+1}_date'] = ''
|
| 245 |
+
|
| 246 |
+
clean_row['languages'] = ', '.join([lang.get('title', '') for lang in row.get('languages', [])])
|
| 247 |
+
clean_row['skills'] = ', '.join([skill.get('title', '') for skill in row.get('skills', [])[:10]])
|
| 248 |
+
|
| 249 |
+
writer.writerow(clean_row)
|
| 250 |
+
|
| 251 |
+
return output_file
|
| 252 |
|
| 253 |
preview_button.click(lambda x: preview_csv(x, scrape_function), inputs=input_file, outputs=preview)
|
| 254 |
clear_preview_button.click(clear_preview, inputs=None, outputs=preview)
|