WebScr / app.py
JaySMTA's picture
Update app.py
e127f06 verified
Raw History Blame
5.95 kB
import os
import asyncio
import aiohttp
import csv
from typing import List, Dict
import gradio as gr
import logging
logging.basicConfig(level=logging.INFO)
# Get environment variables
LINKEDIN_API_KEY = os.environ.get("LINKEDIN_API_KEY")
FACEBOOK_API_KEY = os.environ.get("FACEBOOK_API_KEY")
LINKEDIN_API_ENDPOINT = os.environ.get("LINKEDIN_API_ENDPOINT")
FACEBOOK_API_ENDPOINT = os.environ.get("FACEBOOK_API_ENDPOINT")
# Rate limiting configuration
RATE_LIMIT = 1 # Requests per second
MAX_TOKENS = 10 # Maximum tokens in the bucket
class TokenBucket:
def __init__(self, rate, capacity):
self.rate = rate
self.capacity = capacity
self.tokens = capacity
self.last_update = asyncio.get_event_loop().time()
async def get_token(self):
now = asyncio.get_event_loop().time()
elapsed_time = now - self.last_update
self.last_update = now
new_tokens = elapsed_time * self.rate
self.tokens = min(self.tokens + new_tokens, self.capacity)
if self.tokens >= 1:
self.tokens -= 1
return True
return False
# Create token bucket instances for rate limiting
linkedin_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS)
facebook_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS)
async def rate_limited_api_call(url, headers):
while True:
if "linkedin" in url:
if await linkedin_token_bucket.get_token():
logging.info(f"Making LinkedIn API request: URL={url}, Headers={headers}")
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers) as response:
return await response.json()
elif "facebook" in url:
if await facebook_token_bucket.get_token():
logging.info(f"Making Facebook API request: URL={url}, Headers={headers}")
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers) as response:
return await response.json()
await asyncio.sleep(0.1) # Wait before retrying
async def scrape_linkedin_profile(profile_url: str) -> Dict:
try:
headers = {"Authorization": f"Bearer {LINKEDIN_API_KEY}"}
data = await rate_limited_api_call(f"{LINKEDIN_API_ENDPOINT}?url={profile_url}", headers)
return {
"Name": data.get("name", ""),
"Number of Followers": data.get("num_followers", ""),
"Number of Connections": data.get("num_connections", ""),
"Age": data.get("age", ""),
"Current Role": data.get("current_role", ""),
"Sex": data.get("sex", ""),
"Place of Living": data.get("place_of_living", ""),
}
except Exception as e:
return {"Error": str(e)}
async def scrape_facebook_profile(profile_url: str) -> Dict:
try:
headers = {"Authorization": f"Bearer {FACEBOOK_API_KEY}"}
data = await rate_limited_api_call(f"{FACEBOOK_API_ENDPOINT}?url={profile_url}", headers)
return {
"Full Name": data.get("full_name", ""),
"Location": data.get("location", ""),
"Phone": data.get("phone", ""),
"Email": data.get("email", ""),
"Follower Count": data.get("follower_count", ""),
"Friend Count": data.get("friend_count", ""),
}
except Exception as e:
return {"Error": str(e)}
def clean_text(text: str) -> str:
return text.replace(",", " ").replace("\n", " ").strip()
async def process_csv(file_path: str, scraper_func) -> List[Dict]:
results = []
async with aiohttp.ClientSession() as session:
with open(file_path, "r") as file:
reader = csv.reader(file)
next(reader) # Skip header row
urls = [row[0] for row in reader]
scrape_tasks = [scraper_func(url) for url in urls]
results = await asyncio.gather(*scrape_tasks)
with open("output.csv", "w", newline="") as file:
writer = csv.DictWriter(file, fieldnames=results[0].keys())
writer.writeheader()
for result in results:
writer.writerow({k: clean_text(str(v)) for k, v in result.items()})
return results
async def preview_csv(file_path: str, scraper_func):
results = await process_csv(file_path, scraper_func)
return gr.DataFrame.from_dict(results)
with gr.Blocks() as demo:
linkedin_inputs = gr.Textbox(label="LinkedIn Profile URLs (one per line)")
with gr.Tab("LinkedIn"):
linkedin_csv_file = gr.File(label="Upload CSV", type="filepath")
linkedin_submit_btn = gr.Button("Scrape LinkedIn Profiles")
linkedin_preview_btn = gr.Button("Preview Results")
linkedin_preview_output = gr.Dataframe(label="Preview")
linkedin_scrape_output = gr.Dataframe(label="Scraped Data", visible=False)
facebook_inputs = gr.Textbox(label="Facebook Profile URLs (one per line)")
with gr.Tab("Facebook"):
facebook_csv_file = gr.File(label="Upload CSV", type="filepath")
facebook_submit_btn = gr.Button("Scrape Facebook Profiles")
facebook_preview_btn = gr.Button("Preview Results")
facebook_preview_output = gr.Dataframe(label="Preview")
facebook_scrape_output = gr.Dataframe(label="Scraped Data", visible=False)
linkedin_submit_btn.click(process_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_scrape_output)
linkedin_preview_btn.click(preview_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_preview_output)
facebook_submit_btn.click(process_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_scrape_output)
facebook_preview_btn.click(preview_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_preview_output)
if __name__ == "__main__":
demo.launch()