import os import asyncio import aiohttp import csv from typing import List, Dict import gradio as gr import logging logging.basicConfig(level=logging.INFO) # Get environment variables LINKEDIN_API_KEY = os.environ.get("LINKEDIN_API_KEY") FACEBOOK_API_KEY = os.environ.get("FACEBOOK_API_KEY") LINKEDIN_API_ENDPOINT = os.environ.get("LINKEDIN_API_ENDPOINT") FACEBOOK_API_ENDPOINT = os.environ.get("FACEBOOK_API_ENDPOINT") # Rate limiting configuration RATE_LIMIT = 1 # Requests per second MAX_TOKENS = 10 # Maximum tokens in the bucket class TokenBucket: def __init__(self, rate, capacity): self.rate = rate self.capacity = capacity self.tokens = capacity self.last_update = asyncio.get_event_loop().time() async def get_token(self): now = asyncio.get_event_loop().time() elapsed_time = now - self.last_update self.last_update = now new_tokens = elapsed_time * self.rate self.tokens = min(self.tokens + new_tokens, self.capacity) if self.tokens >= 1: self.tokens -= 1 return True return False # Create token bucket instances for rate limiting linkedin_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS) facebook_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS) async def rate_limited_api_call(url, headers): while True: if "linkedin" in url: if await linkedin_token_bucket.get_token(): logging.info(f"Making LinkedIn API request: URL={url}, Headers={headers}") async with aiohttp.ClientSession() as session: async with session.get(url, headers=headers) as response: return await response.json() elif "facebook" in url: if await facebook_token_bucket.get_token(): logging.info(f"Making Facebook API request: URL={url}, Headers={headers}") async with aiohttp.ClientSession() as session: async with session.get(url, headers=headers) as response: return await response.json() await asyncio.sleep(0.1) # Wait before retrying async def scrape_linkedin_profile(profile_url: str) -> Dict: try: headers = {"Authorization": f"Bearer {LINKEDIN_API_KEY}"} data = await rate_limited_api_call(f"{LINKEDIN_API_ENDPOINT}?url={profile_url}", headers) return { "Name": data.get("name", ""), "Number of Followers": data.get("num_followers", ""), "Number of Connections": data.get("num_connections", ""), "Age": data.get("age", ""), "Current Role": data.get("current_role", ""), "Sex": data.get("sex", ""), "Place of Living": data.get("place_of_living", ""), } except Exception as e: return {"Error": str(e)} async def scrape_facebook_profile(profile_url: str) -> Dict: try: headers = {"Authorization": f"Bearer {FACEBOOK_API_KEY}"} data = await rate_limited_api_call(f"{FACEBOOK_API_ENDPOINT}?url={profile_url}", headers) return { "Full Name": data.get("full_name", ""), "Location": data.get("location", ""), "Phone": data.get("phone", ""), "Email": data.get("email", ""), "Follower Count": data.get("follower_count", ""), "Friend Count": data.get("friend_count", ""), } except Exception as e: return {"Error": str(e)} def clean_text(text: str) -> str: return text.replace(",", " ").replace("\n", " ").strip() async def process_csv(file_path: str, scraper_func) -> List[Dict]: results = [] async with aiohttp.ClientSession() as session: with open(file_path, "r") as file: reader = csv.reader(file) next(reader) # Skip header row urls = [row[0] for row in reader] scrape_tasks = [scraper_func(url) for url in urls] results = await asyncio.gather(*scrape_tasks) with open("output.csv", "w", newline="") as file: writer = csv.DictWriter(file, fieldnames=results[0].keys()) writer.writeheader() for result in results: writer.writerow({k: clean_text(str(v)) for k, v in result.items()}) return results async def preview_csv(file_path: str, scraper_func): results = await process_csv(file_path, scraper_func) return gr.DataFrame.from_dict(results) with gr.Blocks() as demo: linkedin_inputs = gr.Textbox(label="LinkedIn Profile URLs (one per line)") with gr.Tab("LinkedIn"): linkedin_csv_file = gr.File(label="Upload CSV", type="filepath") linkedin_submit_btn = gr.Button("Scrape LinkedIn Profiles") linkedin_preview_btn = gr.Button("Preview Results") linkedin_preview_output = gr.Dataframe(label="Preview") linkedin_scrape_output = gr.Dataframe(label="Scraped Data", visible=False) facebook_inputs = gr.Textbox(label="Facebook Profile URLs (one per line)") with gr.Tab("Facebook"): facebook_csv_file = gr.File(label="Upload CSV", type="filepath") facebook_submit_btn = gr.Button("Scrape Facebook Profiles") facebook_preview_btn = gr.Button("Preview Results") facebook_preview_output = gr.Dataframe(label="Preview") facebook_scrape_output = gr.Dataframe(label="Scraped Data", visible=False) linkedin_submit_btn.click(process_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_scrape_output) linkedin_preview_btn.click(preview_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_preview_output) facebook_submit_btn.click(process_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_scrape_output) facebook_preview_btn.click(preview_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_preview_output) if __name__ == "__main__": demo.launch()