File size: 5,946 Bytes
d5b46b1 c7bd162 d5b46b1 c7bd162 d5b46b1 d130f30 d5b46b1 6290c5a c7bd162 d5b46b1 c7bd162 d5b46b1 c7bd162 d5b46b1 c7bd162 d5b46b1 d130f30 294970a d5b46b1 32a6e9e d5b46b1 abd564d d5b46b1 abd564d d5b46b1 abd564d d5b46b1 32a6e9e d5b46b1 32a6e9e d5b46b1 d130f30 d5b46b1 0b6f916 d5b46b1 9ec997a d5b46b1 e127f06 0b6f916 91b51b3 e127f06 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | import os
import asyncio
import aiohttp
import csv
from typing import List, Dict
import gradio as gr
import logging
logging.basicConfig(level=logging.INFO)
# Get environment variables
LINKEDIN_API_KEY = os.environ.get("LINKEDIN_API_KEY")
FACEBOOK_API_KEY = os.environ.get("FACEBOOK_API_KEY")
LINKEDIN_API_ENDPOINT = os.environ.get("LINKEDIN_API_ENDPOINT")
FACEBOOK_API_ENDPOINT = os.environ.get("FACEBOOK_API_ENDPOINT")
# Rate limiting configuration
RATE_LIMIT = 1 # Requests per second
MAX_TOKENS = 10 # Maximum tokens in the bucket
class TokenBucket:
def __init__(self, rate, capacity):
self.rate = rate
self.capacity = capacity
self.tokens = capacity
self.last_update = asyncio.get_event_loop().time()
async def get_token(self):
now = asyncio.get_event_loop().time()
elapsed_time = now - self.last_update
self.last_update = now
new_tokens = elapsed_time * self.rate
self.tokens = min(self.tokens + new_tokens, self.capacity)
if self.tokens >= 1:
self.tokens -= 1
return True
return False
# Create token bucket instances for rate limiting
linkedin_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS)
facebook_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS)
async def rate_limited_api_call(url, headers):
while True:
if "linkedin" in url:
if await linkedin_token_bucket.get_token():
logging.info(f"Making LinkedIn API request: URL={url}, Headers={headers}")
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers) as response:
return await response.json()
elif "facebook" in url:
if await facebook_token_bucket.get_token():
logging.info(f"Making Facebook API request: URL={url}, Headers={headers}")
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers) as response:
return await response.json()
await asyncio.sleep(0.1) # Wait before retrying
async def scrape_linkedin_profile(profile_url: str) -> Dict:
try:
headers = {"Authorization": f"Bearer {LINKEDIN_API_KEY}"}
data = await rate_limited_api_call(f"{LINKEDIN_API_ENDPOINT}?url={profile_url}", headers)
return {
"Name": data.get("name", ""),
"Number of Followers": data.get("num_followers", ""),
"Number of Connections": data.get("num_connections", ""),
"Age": data.get("age", ""),
"Current Role": data.get("current_role", ""),
"Sex": data.get("sex", ""),
"Place of Living": data.get("place_of_living", ""),
}
except Exception as e:
return {"Error": str(e)}
async def scrape_facebook_profile(profile_url: str) -> Dict:
try:
headers = {"Authorization": f"Bearer {FACEBOOK_API_KEY}"}
data = await rate_limited_api_call(f"{FACEBOOK_API_ENDPOINT}?url={profile_url}", headers)
return {
"Full Name": data.get("full_name", ""),
"Location": data.get("location", ""),
"Phone": data.get("phone", ""),
"Email": data.get("email", ""),
"Follower Count": data.get("follower_count", ""),
"Friend Count": data.get("friend_count", ""),
}
except Exception as e:
return {"Error": str(e)}
def clean_text(text: str) -> str:
return text.replace(",", " ").replace("\n", " ").strip()
async def process_csv(file_path: str, scraper_func) -> List[Dict]:
results = []
async with aiohttp.ClientSession() as session:
with open(file_path, "r") as file:
reader = csv.reader(file)
next(reader) # Skip header row
urls = [row[0] for row in reader]
scrape_tasks = [scraper_func(url) for url in urls]
results = await asyncio.gather(*scrape_tasks)
with open("output.csv", "w", newline="") as file:
writer = csv.DictWriter(file, fieldnames=results[0].keys())
writer.writeheader()
for result in results:
writer.writerow({k: clean_text(str(v)) for k, v in result.items()})
return results
async def preview_csv(file_path: str, scraper_func):
results = await process_csv(file_path, scraper_func)
return gr.DataFrame.from_dict(results)
with gr.Blocks() as demo:
linkedin_inputs = gr.Textbox(label="LinkedIn Profile URLs (one per line)")
with gr.Tab("LinkedIn"):
linkedin_csv_file = gr.File(label="Upload CSV", type="filepath")
linkedin_submit_btn = gr.Button("Scrape LinkedIn Profiles")
linkedin_preview_btn = gr.Button("Preview Results")
linkedin_preview_output = gr.Dataframe(label="Preview")
linkedin_scrape_output = gr.Dataframe(label="Scraped Data", visible=False)
facebook_inputs = gr.Textbox(label="Facebook Profile URLs (one per line)")
with gr.Tab("Facebook"):
facebook_csv_file = gr.File(label="Upload CSV", type="filepath")
facebook_submit_btn = gr.Button("Scrape Facebook Profiles")
facebook_preview_btn = gr.Button("Preview Results")
facebook_preview_output = gr.Dataframe(label="Preview")
facebook_scrape_output = gr.Dataframe(label="Scraped Data", visible=False)
linkedin_submit_btn.click(process_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_scrape_output)
linkedin_preview_btn.click(preview_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_preview_output)
facebook_submit_btn.click(process_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_scrape_output)
facebook_preview_btn.click(preview_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_preview_output)
if __name__ == "__main__":
demo.launch() |