Update app.py
Browse files
app.py
CHANGED
|
@@ -72,7 +72,7 @@ async def scrape_linkedin_profile(session: ClientSession, bucket: TokenBucket, u
|
|
| 72 |
|
| 73 |
if not api_key:
|
| 74 |
logging.error("LinkedIn API key not found in environment variables")
|
| 75 |
-
return
|
| 76 |
|
| 77 |
headers = {
|
| 78 |
'Authorization': f'Bearer {api_key}',
|
|
@@ -100,18 +100,18 @@ async def scrape_linkedin_profile(session: ClientSession, bucket: TokenBucket, u
|
|
| 100 |
else:
|
| 101 |
error_message = f"API error for LinkedIn profile {url}: {result.get('message', 'Unknown error')}"
|
| 102 |
logging.error(error_message)
|
| 103 |
-
return error_message
|
| 104 |
except Exception as e:
|
| 105 |
error_message = f"Error scraping LinkedIn profile {url}: {str(e)}"
|
| 106 |
logging.error(error_message)
|
| 107 |
-
return error_message
|
| 108 |
|
| 109 |
async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
|
| 110 |
api_key = await get_secret('FACEBOOK_API_KEY')
|
| 111 |
api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
|
| 112 |
if not api_key or not api_endpoint:
|
| 113 |
logging.error("Facebook API key or endpoint not found in environment variables")
|
| 114 |
-
return
|
| 115 |
|
| 116 |
headers = {
|
| 117 |
'Authorization': f'Bearer {api_key}',
|
|
@@ -134,11 +134,13 @@ async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, u
|
|
| 134 |
"Friend Count": result.get('friend_count', '')
|
| 135 |
}
|
| 136 |
else:
|
| 137 |
-
|
| 138 |
-
|
|
|
|
| 139 |
except Exception as e:
|
| 140 |
-
|
| 141 |
-
|
|
|
|
| 142 |
|
| 143 |
async def process_csv(csv_file, scrape_function):
|
| 144 |
try:
|
|
@@ -194,7 +196,7 @@ def create_interface(name, scrape_function):
|
|
| 194 |
|
| 195 |
def scrape_and_download(csv_file):
|
| 196 |
result = asyncio.run(process_csv(csv_file, scrape_function))
|
| 197 |
-
if result.startswith("Error"):
|
| 198 |
return None
|
| 199 |
data = json.loads(result)
|
| 200 |
output_file = f"{scrape_function.__name__}_scraped_data.csv"
|
|
@@ -205,8 +207,11 @@ def create_interface(name, scrape_function):
|
|
| 205 |
with open(output_file, 'w', newline='', encoding='utf-8') as f:
|
| 206 |
writer = csv.writer(f)
|
| 207 |
for item in data:
|
| 208 |
-
|
| 209 |
-
writer.writerow([
|
|
|
|
|
|
|
|
|
|
| 210 |
writer.writerow([]) # Empty row between profiles
|
| 211 |
|
| 212 |
return output_file
|
|
|
|
| 72 |
|
| 73 |
if not api_key:
|
| 74 |
logging.error("LinkedIn API key not found in environment variables")
|
| 75 |
+
return {"Error": "LinkedIn API key not found"}
|
| 76 |
|
| 77 |
headers = {
|
| 78 |
'Authorization': f'Bearer {api_key}',
|
|
|
|
| 100 |
else:
|
| 101 |
error_message = f"API error for LinkedIn profile {url}: {result.get('message', 'Unknown error')}"
|
| 102 |
logging.error(error_message)
|
| 103 |
+
return {"Error": error_message}
|
| 104 |
except Exception as e:
|
| 105 |
error_message = f"Error scraping LinkedIn profile {url}: {str(e)}"
|
| 106 |
logging.error(error_message)
|
| 107 |
+
return {"Error": error_message}
|
| 108 |
|
| 109 |
async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
|
| 110 |
api_key = await get_secret('FACEBOOK_API_KEY')
|
| 111 |
api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
|
| 112 |
if not api_key or not api_endpoint:
|
| 113 |
logging.error("Facebook API key or endpoint not found in environment variables")
|
| 114 |
+
return {"Error": "Facebook API key or endpoint not found"}
|
| 115 |
|
| 116 |
headers = {
|
| 117 |
'Authorization': f'Bearer {api_key}',
|
|
|
|
| 134 |
"Friend Count": result.get('friend_count', '')
|
| 135 |
}
|
| 136 |
else:
|
| 137 |
+
error_message = f"API error for Facebook profile {url}: {result.get('error')}"
|
| 138 |
+
logging.error(error_message)
|
| 139 |
+
return {"Error": error_message}
|
| 140 |
except Exception as e:
|
| 141 |
+
error_message = f"Error scraping Facebook profile {url}: {str(e)}"
|
| 142 |
+
logging.error(error_message)
|
| 143 |
+
return {"Error": error_message}
|
| 144 |
|
| 145 |
async def process_csv(csv_file, scrape_function):
|
| 146 |
try:
|
|
|
|
| 196 |
|
| 197 |
def scrape_and_download(csv_file):
|
| 198 |
result = asyncio.run(process_csv(csv_file, scrape_function))
|
| 199 |
+
if isinstance(result, str) and result.startswith("Error"):
|
| 200 |
return None
|
| 201 |
data = json.loads(result)
|
| 202 |
output_file = f"{scrape_function.__name__}_scraped_data.csv"
|
|
|
|
| 207 |
with open(output_file, 'w', newline='', encoding='utf-8') as f:
|
| 208 |
writer = csv.writer(f)
|
| 209 |
for item in data:
|
| 210 |
+
if "Error" in item:
|
| 211 |
+
writer.writerow(["Error", clean_text(item["Error"])])
|
| 212 |
+
else:
|
| 213 |
+
for key, value in item.items():
|
| 214 |
+
writer.writerow([f"{key}: {clean_text(value)}"])
|
| 215 |
writer.writerow([]) # Empty row between profiles
|
| 216 |
|
| 217 |
return output_file
|