File size: 11,145 Bytes
a243250 ac078a6 a243250 c7bd162 a243250 c7bd162 d5b46b1 a243250 6290c5a c7bd162 9af716e 5aea171 c7bd162 a243250 c7bd162 a243250 c7bd162 a243250 06a1d93 852ccee abd564d 06a1d93 a243250 6557cf4 852ccee 06a1d93 6557cf4 06a1d93 852ccee 06a1d93 09e710f 06a1d93 a243250 09e710f a243250 06a1d93 ac078a6 a243250 6557cf4 a243250 6557cf4 0b6f916 a243250 6557cf4 a243250 09e710f 06a1d93 a243250 a622a0b a243250 9ec997a a243250 b90ef62 a243250 b90ef62 a243250 b90ef62 a243250 e127f06 a243250 b90ef62 a243250 0b6f916 91b51b3 0f52ebb a243250 9af716e a243250 9af716e a243250 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 | import os
import csv
import urllib.parse
import json
import logging
import asyncio
import aiohttp
import time
import random
from asyncio import Semaphore
from aiohttp import ClientSession
from typing import Dict, Any
import gradio as gr
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
# Rate limiting configuration
LINKEDIN_RATE_LIMIT = 1
FACEBOOK_RATE_LIMIT = 1
RATE_LIMIT_PERIOD = 1
MAX_TOKENS = 1
class TokenBucket:
def __init__(self, rate: float, max_tokens: int):
self.rate = rate
self.max_tokens = max_tokens
self.tokens = max_tokens
self.last_refill = time.monotonic()
self.lock = asyncio.Lock()
async def get_token(self):
async with self.lock:
now = time.monotonic()
time_passed = now - self.last_refill
self.tokens = min(self.max_tokens, self.tokens + time_passed * self.rate)
self.last_refill = now
if self.tokens >= 1:
self.tokens -= 1
return True
else:
return False
async def wait_for_token(self):
while not await self.get_token():
await asyncio.sleep(1 / self.rate)
async def get_secret(secret_name: str) -> str:
return os.environ.get(secret_name)
async def rate_limited_api_call(session: ClientSession, bucket: TokenBucket, url: str, headers: Dict[str, str], params: Dict[str, Any] = None, data: Dict[str, Any] = None, method: str = 'GET') -> Dict[str, Any]:
max_retries = 5
base_delay = 5 # seconds
for attempt in range(max_retries):
await bucket.wait_for_token()
# Add a small random delay before each API call
await asyncio.sleep(random.uniform(0.1, 0.5))
try:
if method == 'GET':
async with session.get(url, headers=headers, params=params) as response:
response.raise_for_status()
return await response.json()
elif method == 'POST':
async with session.post(url, headers=headers, params=params, json=data) as response:
response.raise_for_status()
return await response.json()
except aiohttp.ClientResponseError as e:
if e.status == 503 and attempt < max_retries - 1:
retry_delay = base_delay * (2 ** attempt) + random.uniform(0, 1) # Exponential backoff with jitter
logging.warning(f"Attempt {attempt + 1} failed with 503 error. Retrying in {retry_delay:.2f} seconds...")
await asyncio.sleep(retry_delay)
else:
logging.error(f"API call failed: {str(e)}")
raise
except Exception as e:
logging.error(f"Unexpected error in API call: {str(e)}")
raise
raise Exception(f"Failed after {max_retries} attempts")
async def scrape_linkedin_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
api_key = await get_secret('LINKEDIN_API_KEY')
api_endpoint = "https://zylalabs.com/api/1511/linkedin+profile+scraper+api/5771/profile+scraper"
if not api_key:
logging.error("LinkedIn API key not found in environment variables")
return {"Error": "LinkedIn API key not found"}
headers = {
'Authorization': f'Bearer {api_key}',
'Content-Type': 'application/json'
}
data = {
"link": url
}
try:
result = await rate_limited_api_call(session, bucket, api_endpoint, headers, data=data, method='POST')
if 'data' in result:
profile_data = result['data']
return {
"Name": profile_data.get('fullName', ''),
"Number of Followers": profile_data.get('followers', ''),
"Number of Connections": profile_data.get('connections', ''),
"Age": profile_data.get('age', ''),
"Current Role": profile_data.get('headline', ''),
"Sex": profile_data.get('gender', ''),
"Place of living": profile_data.get('addressWithCountry', '')
}
else:
error_message = f"API error for LinkedIn profile {url}: {result.get('message', 'Unknown error')}"
logging.error(error_message)
return {"Error": error_message}
except Exception as e:
error_message = f"Error scraping LinkedIn profile {url}: {str(e)}"
logging.error(error_message)
return {"Error": error_message}
async def scrape_facebook_profile(session: ClientSession, bucket: TokenBucket, url: str) -> Dict[str, Any]:
api_key = await get_secret('FACEBOOK_API_KEY')
api_endpoint = await get_secret('FACEBOOK_API_ENDPOINT')
if not api_key or not api_endpoint:
logging.error("Facebook API key or endpoint not found in environment variables")
return {"Error": "Facebook API key or endpoint not found in environment variables"}
# Trim any whitespace and encode the API endpoint
api_endpoint = api_endpoint.strip()
api_endpoint = urllib.parse.quote(api_endpoint, safe=':/')
logging.info(f"Facebook API Endpoint: {api_endpoint}")
headers = {
'Authorization': f'Bearer {api_key}'
}
# Extract the profile ID or username from the URL
profile_id = url.split('/')[-1].strip()
# Add the id as a query parameter
params = {'id': profile_id}
try:
result = await rate_limited_api_call(session, bucket, api_endpoint, headers, params=params, method='POST')
if 'error' not in result:
return {
"Full Name": result.get('name', ''),
"Location": result.get('location', ''),
"Phone": result.get('phone', ''),
"Email": result.get('email', ''),
"Follower Count": result.get('follower_count', ''),
"Friend Count": result.get('friend_count', '')
}
else:
error_message = f"API error for Facebook profile {url}: {result.get('error')}"
logging.error(error_message)
return {"Error": error_message}
except aiohttp.ClientResponseError as e:
error_message = f"HTTP error when scraping Facebook profile {url}: {e.status} {e.message}"
logging.error(error_message)
return {"Error": error_message}
except Exception as e:
error_message = f"Error scraping Facebook profile {url}: {str(e)}"
logging.error(error_message)
return {"Error": error_message}
async def process_csv(csv_file, scrape_function):
try:
with open(csv_file.name, 'r') as file:
reader = csv.reader(file)
profile_urls = [row[0] for row in reader if row]
linkedin_bucket = TokenBucket(LINKEDIN_RATE_LIMIT, MAX_TOKENS)
facebook_bucket = TokenBucket(FACEBOOK_RATE_LIMIT, MAX_TOKENS)
async with aiohttp.ClientSession() as session:
tasks = []
for url in profile_urls:
if 'linkedin.com' in url:
tasks.append(scrape_function(session, linkedin_bucket, url))
elif 'facebook.com' in url:
tasks.append(scrape_function(session, facebook_bucket, url))
else:
tasks.append(asyncio.create_task(asyncio.sleep(0))) # Dummy task for unsupported URLs
results = await asyncio.gather(*tasks)
return json.dumps([r for r in results if r is not None], indent=2)
except Exception as e:
logging.error(f"Error processing CSV file: {str(e)}")
return f"Error processing CSV file: {str(e)}"
async def preview_csv(csv_file, scrape_function):
try:
with open(csv_file.name, 'r') as file:
reader = csv.reader(file)
profile_urls = [row[0] for row in reader if row]
if not profile_urls:
return "No URLs found in the CSV file."
preview_url = profile_urls[0]
bucket = TokenBucket(LINKEDIN_RATE_LIMIT if 'linkedin.com' in preview_url else FACEBOOK_RATE_LIMIT, MAX_TOKENS)
async with aiohttp.ClientSession() as session:
preview_data = await scrape_function(session, bucket, preview_url)
if preview_data:
return json.dumps(preview_data, indent=2)
else:
return "Error fetching preview data."
except Exception as e:
logging.error(f"Error previewing CSV file: {str(e)}")
return f"Error previewing CSV file: {str(e)}"
def create_interface(name, scrape_function):
input_file = gr.File(label=f"Upload CSV file with {name} profile URLs")
preview = gr.Textbox(label="Preview (First Profile)", max_lines=10)
output = gr.Textbox(label=f"Scraped {name} Data")
preview_button = gr.Button("Preview")
clear_preview_button = gr.Button("Clear Preview")
scrape_button = gr.Button(f"Scrape {name}")
download = gr.File(label="Download Scraped Data as CSV")
def clear_preview():
return ""
def scrape_and_download(csv_file):
result = asyncio.run(process_csv(csv_file, scrape_function))
if isinstance(result, str) and result.startswith("Error"):
return None
data = json.loads(result)
output_file = f"{scrape_function.__name__}_scraped_data.csv"
def clean_text(text):
return str(text).replace('\n', ' ').strip()
with open(output_file, 'w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
for item in data:
if "Error" in item:
writer.writerow(["Error", clean_text(item["Error"])])
else:
for key, value in item.items():
writer.writerow([f"{key}: {clean_text(value)}"])
writer.writerow([]) # Empty row between profiles
return output_file
preview_button.click(lambda x: asyncio.run(preview_csv(x, scrape_function)), inputs=input_file, outputs=preview)
clear_preview_button.click(clear_preview, inputs=None, outputs=preview)
scrape_button.click(lambda x: asyncio.run(process_csv(x, scrape_function)), inputs=input_file, outputs=output)
scrape_button.click(scrape_and_download, inputs=input_file, outputs=download)
return input_file, preview, output, download
def create_app():
with gr.Blocks() as demo:
gr.Markdown("# Multi-functional Data Processing Application")
with gr.Tab("LinkedIn Scraper"):
create_interface("LinkedIn", scrape_linkedin_profile)
with gr.Tab("Facebook Scraper"):
create_interface("Facebook", scrape_facebook_profile)
return demo
# Main application
app = create_app()
if __name__ == "__main__":
logging.info(f"Facebook API Endpoint:{os.environ.get('FACEBOOK_API_ENDPOINT', 'Not set')} ")
if "SPACE_ID" in os.environ:
app.launch()
else:
app.launch(share=True) |