JaySMTA commited on
Commit
23a5499
·
verified ·
1 Parent(s): 25a19e6

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +16 -11
app.py CHANGED
@@ -72,7 +72,7 @@ async def scrape_linkedin_profile(session: ClientSession, bucket: TokenBucket, u
72
 
73
  if not api_key:
74
  logging.error("LinkedIn API key not found in environment variables")
75
- return None
76
 
77
  headers = {
78
  'Authorization': f'Bearer {api_key}',
@@ -100,18 +100,18 @@ async def scrape_linkedin_profile(session: ClientSession, bucket: TokenBucket, u
100
  else:
101
  error_message = f"API error for LinkedIn profile {url}: {result.get('message', 'Unknown error')}"
102
  logging.error(error_message)
103
- return error_message
104
  except Exception as e:
105
  error_message = f"Error scraping LinkedIn profile {url}: {str(e)}"
106
  logging.error(error_message)
107
- return error_message
108
 
109
  async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
110
  api_key = await get_secret('FACEBOOK_API_KEY')
111
  api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
112
  if not api_key or not api_endpoint:
113
  logging.error("Facebook API key or endpoint not found in environment variables")
114
- return None
115
 
116
  headers = {
117
  'Authorization': f'Bearer {api_key}',
@@ -134,11 +134,13 @@ async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, u
134
  "Friend Count": result.get('friend_count', '')
135
  }
136
  else:
137
- logging.error(f"API error for Facebook profile {url}: {result.get('error')}")
138
- return None
 
139
  except Exception as e:
140
- logging.error(f"Error scraping Facebook profile {url}: {str(e)}")
141
- return None
 
142
 
143
  async def process_csv(csv_file, scrape_function):
144
  try:
@@ -194,7 +196,7 @@ def create_interface(name, scrape_function):
194
 
195
  def scrape_and_download(csv_file):
196
  result = asyncio.run(process_csv(csv_file, scrape_function))
197
- if result.startswith("Error"):
198
  return None
199
  data = json.loads(result)
200
  output_file = f"{scrape_function.__name__}_scraped_data.csv"
@@ -205,8 +207,11 @@ def create_interface(name, scrape_function):
205
  with open(output_file, 'w', newline='', encoding='utf-8') as f:
206
  writer = csv.writer(f)
207
  for item in data:
208
- for key, value in item.items():
209
- writer.writerow([f"{key}: {clean_text(value)}"])
 
 
 
210
  writer.writerow([]) # Empty row between profiles
211
 
212
  return output_file
 
72
 
73
  if not api_key:
74
  logging.error("LinkedIn API key not found in environment variables")
75
+ return {"Error": "LinkedIn API key not found"}
76
 
77
  headers = {
78
  'Authorization': f'Bearer {api_key}',
 
100
  else:
101
  error_message = f"API error for LinkedIn profile {url}: {result.get('message', 'Unknown error')}"
102
  logging.error(error_message)
103
+ return {"Error": error_message}
104
  except Exception as e:
105
  error_message = f"Error scraping LinkedIn profile {url}: {str(e)}"
106
  logging.error(error_message)
107
+ return {"Error": error_message}
108
 
109
  async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
110
  api_key = await get_secret('FACEBOOK_API_KEY')
111
  api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
112
  if not api_key or not api_endpoint:
113
  logging.error("Facebook API key or endpoint not found in environment variables")
114
+ return {"Error": "Facebook API key or endpoint not found"}
115
 
116
  headers = {
117
  'Authorization': f'Bearer {api_key}',
 
134
  "Friend Count": result.get('friend_count', '')
135
  }
136
  else:
137
+ error_message = f"API error for Facebook profile {url}: {result.get('error')}"
138
+ logging.error(error_message)
139
+ return {"Error": error_message}
140
  except Exception as e:
141
+ error_message = f"Error scraping Facebook profile {url}: {str(e)}"
142
+ logging.error(error_message)
143
+ return {"Error": error_message}
144
 
145
  async def process_csv(csv_file, scrape_function):
146
  try:
 
196
 
197
  def scrape_and_download(csv_file):
198
  result = asyncio.run(process_csv(csv_file, scrape_function))
199
+ if isinstance(result, str) and result.startswith("Error"):
200
  return None
201
  data = json.loads(result)
202
  output_file = f"{scrape_function.__name__}_scraped_data.csv"
 
207
  with open(output_file, 'w', newline='', encoding='utf-8') as f:
208
  writer = csv.writer(f)
209
  for item in data:
210
+ if "Error" in item:
211
+ writer.writerow(["Error", clean_text(item["Error"])])
212
+ else:
213
+ for key, value in item.items():
214
+ writer.writerow([f"{key}: {clean_text(value)}"])
215
  writer.writerow([]) # Empty row between profiles
216
 
217
  return output_file