JaySMTA commited on
Commit
06a1d93
·
verified ·
1 Parent(s): 5aea171

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +33 -39
app.py CHANGED
@@ -66,52 +66,42 @@ async def rate_limited_api_call(session: ClientSession, bucket: TokenBucket, url
66
  logging.error(f"Unexpected error in API call: {str(e)}")
67
  raise
68
 
69
- async def scrape_linkedin_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
70
- api_key = await get_secret('LINKEDIN_API_KEY')
71
- api_endpoint = "https://zylalabs.com/api/1511/linkedin+profile+scraper+api/5771/profile+scraper"
72
-
73
- if not api_key:
74
- logging.error("LinkedIn API key not found in environment variables")
75
- return {"Error": "LinkedIn API key not found"}
76
-
77
- headers = {
78
- 'Authorization': f'Bearer {api_key}',
79
- 'Content-Type': 'application/json'
80
- }
81
-
82
- data = {
83
- "link": url
84
- }
85
 
86
- try:
87
- result = await rate_limited_api_call(session, bucket, api_endpoint, headers, data=data, method='POST')
88
 
89
- if 'data' in result:
90
- profile_data = result['data']
91
- return {
92
- "Name": profile_data.get('fullName', ''),
93
- "Number of Followers": profile_data.get('followers', ''),
94
- "Number of Connections": profile_data.get('connections', ''),
95
- "Age": profile_data.get('age', ''),
96
- "Current Role": profile_data.get('headline', ''),
97
- "Sex": profile_data.get('gender', ''),
98
- "Place of living": profile_data.get('addressWithCountry', '')
99
- }
100
- else:
101
- error_message = f"API error for LinkedIn profile {url}: {result.get('message', 'Unknown error')}"
102
- logging.error(error_message)
103
- return {"Error": error_message}
104
- except Exception as e:
105
- error_message = f"Error scraping LinkedIn profile {url}: {str(e)}"
106
- logging.error(error_message)
107
- return {"Error": error_message}
 
 
108
 
109
  async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
110
  api_key = await get_secret('FACEBOOK_API_KEY')
111
  api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
112
  if not api_key or not api_endpoint:
113
  logging.error("Facebook API key or endpoint not found in environment variables")
114
- return {"Error": "Facebook API key or endpoint not found"}
 
115
 
116
  headers = {
117
  'Authorization': f'Bearer {api_key}',
@@ -136,7 +126,11 @@ async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, u
136
  else:
137
  error_message = f"API error for Facebook profile {url}: {result.get('error')}"
138
  logging.error(error_message)
139
- return {"Error": error_message}
 
 
 
 
140
  except Exception as e:
141
  error_message = f"Error scraping Facebook profile {url}: {str(e)}"
142
  logging.error(error_message)
 
66
  logging.error(f"Unexpected error in API call: {str(e)}")
67
  raise
68
 
69
+ async def rate_limited_api_call(session: ClientSession, bucket: TokenBucket, url: str, headers: Dict[str, str], params: Dict[str, Any] = None, data: Dict[str, Any] = None, method: str = 'GET') -> Dict[str, Any]:
70
+ max_retries = 3
71
+ retry_delay = 5 # seconds
 
 
 
 
 
 
 
 
 
 
 
 
 
72
 
73
+ for attempt in range(max_retries):
74
+ await bucket.wait_for_token()
75
 
76
+ try:
77
+ if method == 'GET':
78
+ async with session.get(url, headers=headers, params=params) as response:
79
+ response.raise_for_status()
80
+ return await response.json()
81
+ elif method == 'POST':
82
+ async with session.post(url, headers=headers, json=data) as response:
83
+ response.raise_for_status()
84
+ return await response.json()
85
+ except aiohttp.ClientResponseError as e:
86
+ if e.status == 503 and attempt < max_retries - 1:
87
+ logging.warning(f"Attempt {attempt + 1} failed with 503 error. Retrying in {retry_delay} seconds...")
88
+ await asyncio.sleep(retry_delay)
89
+ else:
90
+ logging.error(f"API call failed: {str(e)}")
91
+ raise
92
+ except Exception as e:
93
+ logging.error(f"Unexpected error in API call: {str(e)}")
94
+ raise
95
+
96
+ raise Exception(f"Failed after {max_retries} attempts")
97
 
98
  async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
99
  api_key = await get_secret('FACEBOOK_API_KEY')
100
  api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
101
  if not api_key or not api_endpoint:
102
  logging.error("Facebook API key or endpoint not found in environment variables")
103
+ return {"Error": "Facebook API key or endpoint not found in environment variables"}
104
+
105
 
106
  headers = {
107
  'Authorization': f'Bearer {api_key}',
 
126
  else:
127
  error_message = f"API error for Facebook profile {url}: {result.get('error')}"
128
  logging.error(error_message)
129
+ return {"Error": error_message}
130
+ except aiohttp.ClientResponseError as e:
131
+ error_message = f"HTTP error when scraping Facebook profile {url}: {e.status} {e.message}"
132
+ logging.error(error_message)
133
+ return {"Error": error_message}
134
  except Exception as e:
135
  error_message = f"Error scraping Facebook profile {url}: {str(e)}"
136
  logging.error(error_message)