Update app.py
Browse files
app.py
CHANGED
|
@@ -66,52 +66,42 @@ async def rate_limited_api_call(session: ClientSession, bucket: TokenBucket, url
|
|
| 66 |
logging.error(f"Unexpected error in API call: {str(e)}")
|
| 67 |
raise
|
| 68 |
|
| 69 |
-
async def
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
if not api_key:
|
| 74 |
-
logging.error("LinkedIn API key not found in environment variables")
|
| 75 |
-
return {"Error": "LinkedIn API key not found"}
|
| 76 |
-
|
| 77 |
-
headers = {
|
| 78 |
-
'Authorization': f'Bearer {api_key}',
|
| 79 |
-
'Content-Type': 'application/json'
|
| 80 |
-
}
|
| 81 |
-
|
| 82 |
-
data = {
|
| 83 |
-
"link": url
|
| 84 |
-
}
|
| 85 |
|
| 86 |
-
|
| 87 |
-
|
| 88 |
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
|
|
|
|
|
|
| 108 |
|
| 109 |
async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
|
| 110 |
api_key = await get_secret('FACEBOOK_API_KEY')
|
| 111 |
api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
|
| 112 |
if not api_key or not api_endpoint:
|
| 113 |
logging.error("Facebook API key or endpoint not found in environment variables")
|
| 114 |
-
return {"Error": "Facebook API key or endpoint not found"}
|
|
|
|
| 115 |
|
| 116 |
headers = {
|
| 117 |
'Authorization': f'Bearer {api_key}',
|
|
@@ -136,7 +126,11 @@ async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, u
|
|
| 136 |
else:
|
| 137 |
error_message = f"API error for Facebook profile {url}: {result.get('error')}"
|
| 138 |
logging.error(error_message)
|
| 139 |
-
return {"Error": error_message}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
except Exception as e:
|
| 141 |
error_message = f"Error scraping Facebook profile {url}: {str(e)}"
|
| 142 |
logging.error(error_message)
|
|
|
|
| 66 |
logging.error(f"Unexpected error in API call: {str(e)}")
|
| 67 |
raise
|
| 68 |
|
| 69 |
+
async def rate_limited_api_call(session: ClientSession, bucket: TokenBucket, url: str, headers: Dict[str, str], params: Dict[str, Any] = None, data: Dict[str, Any] = None, method: str = 'GET') -> Dict[str, Any]:
|
| 70 |
+
max_retries = 3
|
| 71 |
+
retry_delay = 5 # seconds
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
+
for attempt in range(max_retries):
|
| 74 |
+
await bucket.wait_for_token()
|
| 75 |
|
| 76 |
+
try:
|
| 77 |
+
if method == 'GET':
|
| 78 |
+
async with session.get(url, headers=headers, params=params) as response:
|
| 79 |
+
response.raise_for_status()
|
| 80 |
+
return await response.json()
|
| 81 |
+
elif method == 'POST':
|
| 82 |
+
async with session.post(url, headers=headers, json=data) as response:
|
| 83 |
+
response.raise_for_status()
|
| 84 |
+
return await response.json()
|
| 85 |
+
except aiohttp.ClientResponseError as e:
|
| 86 |
+
if e.status == 503 and attempt < max_retries - 1:
|
| 87 |
+
logging.warning(f"Attempt {attempt + 1} failed with 503 error. Retrying in {retry_delay} seconds...")
|
| 88 |
+
await asyncio.sleep(retry_delay)
|
| 89 |
+
else:
|
| 90 |
+
logging.error(f"API call failed: {str(e)}")
|
| 91 |
+
raise
|
| 92 |
+
except Exception as e:
|
| 93 |
+
logging.error(f"Unexpected error in API call: {str(e)}")
|
| 94 |
+
raise
|
| 95 |
+
|
| 96 |
+
raise Exception(f"Failed after {max_retries} attempts")
|
| 97 |
|
| 98 |
async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
|
| 99 |
api_key = await get_secret('FACEBOOK_API_KEY')
|
| 100 |
api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
|
| 101 |
if not api_key or not api_endpoint:
|
| 102 |
logging.error("Facebook API key or endpoint not found in environment variables")
|
| 103 |
+
return {"Error": "Facebook API key or endpoint not found in environment variables"}
|
| 104 |
+
|
| 105 |
|
| 106 |
headers = {
|
| 107 |
'Authorization': f'Bearer {api_key}',
|
|
|
|
| 126 |
else:
|
| 127 |
error_message = f"API error for Facebook profile {url}: {result.get('error')}"
|
| 128 |
logging.error(error_message)
|
| 129 |
+
return {"Error": error_message}
|
| 130 |
+
except aiohttp.ClientResponseError as e:
|
| 131 |
+
error_message = f"HTTP error when scraping Facebook profile {url}: {e.status} {e.message}"
|
| 132 |
+
logging.error(error_message)
|
| 133 |
+
return {"Error": error_message}
|
| 134 |
except Exception as e:
|
| 135 |
error_message = f"Error scraping Facebook profile {url}: {str(e)}"
|
| 136 |
logging.error(error_message)
|