Download app.py from JaySMTA/WebScr: direct link, hf CLI and curl.
- Browser
- Download file 5.95 kB
-
https://huggingface.co/spaces/JaySMTA/WebScr/resolve/e127f0678ed1ec2ff9c06f974169da170b6797d6/app.py
- Command line
-
hf download hf://spaces/JaySMTA/WebScr@e127f0678ed1ec2ff9c06f974169da170b6797d6/app.py
-
curl -L -o app.py https://huggingface.co/spaces/JaySMTA/WebScr/resolve/e127f0678ed1ec2ff9c06f974169da170b6797d6/app.py
5.95 kB
| import os | |
| import asyncio | |
| import aiohttp | |
| import csv | |
| from typing import List, Dict | |
| import gradio as gr | |
| import logging | |
| logging.basicConfig(level=logging.INFO) | |
| # Get environment variables | |
| LINKEDIN_API_KEY = os.environ.get("LINKEDIN_API_KEY") | |
| FACEBOOK_API_KEY = os.environ.get("FACEBOOK_API_KEY") | |
| LINKEDIN_API_ENDPOINT = os.environ.get("LINKEDIN_API_ENDPOINT") | |
| FACEBOOK_API_ENDPOINT = os.environ.get("FACEBOOK_API_ENDPOINT") | |
| # Rate limiting configuration | |
| RATE_LIMIT = 1 # Requests per second | |
| MAX_TOKENS = 10 # Maximum tokens in the bucket | |
| class TokenBucket: | |
| def __init__(self, rate, capacity): | |
| self.rate = rate | |
| self.capacity = capacity | |
| self.tokens = capacity | |
| self.last_update = asyncio.get_event_loop().time() | |
| async def get_token(self): | |
| now = asyncio.get_event_loop().time() | |
| elapsed_time = now - self.last_update | |
| self.last_update = now | |
| new_tokens = elapsed_time * self.rate | |
| self.tokens = min(self.tokens + new_tokens, self.capacity) | |
| if self.tokens >= 1: | |
| self.tokens -= 1 | |
| return True | |
| return False | |
| # Create token bucket instances for rate limiting | |
| linkedin_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS) | |
| facebook_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS) | |
| async def rate_limited_api_call(url, headers): | |
| while True: | |
| if "linkedin" in url: | |
| if await linkedin_token_bucket.get_token(): | |
| logging.info(f"Making LinkedIn API request: URL={url}, Headers={headers}") | |
| async with aiohttp.ClientSession() as session: | |
| async with session.get(url, headers=headers) as response: | |
| return await response.json() | |
| elif "facebook" in url: | |
| if await facebook_token_bucket.get_token(): | |
| logging.info(f"Making Facebook API request: URL={url}, Headers={headers}") | |
| async with aiohttp.ClientSession() as session: | |
| async with session.get(url, headers=headers) as response: | |
| return await response.json() | |
| await asyncio.sleep(0.1) # Wait before retrying | |
| async def scrape_linkedin_profile(profile_url: str) -> Dict: | |
| try: | |
| headers = {"Authorization": f"Bearer {LINKEDIN_API_KEY}"} | |
| data = await rate_limited_api_call(f"{LINKEDIN_API_ENDPOINT}?url={profile_url}", headers) | |
| return { | |
| "Name": data.get("name", ""), | |
| "Number of Followers": data.get("num_followers", ""), | |
| "Number of Connections": data.get("num_connections", ""), | |
| "Age": data.get("age", ""), | |
| "Current Role": data.get("current_role", ""), | |
| "Sex": data.get("sex", ""), | |
| "Place of Living": data.get("place_of_living", ""), | |
| } | |
| except Exception as e: | |
| return {"Error": str(e)} | |
| async def scrape_facebook_profile(profile_url: str) -> Dict: | |
| try: | |
| headers = {"Authorization": f"Bearer {FACEBOOK_API_KEY}"} | |
| data = await rate_limited_api_call(f"{FACEBOOK_API_ENDPOINT}?url={profile_url}", headers) | |
| return { | |
| "Full Name": data.get("full_name", ""), | |
| "Location": data.get("location", ""), | |
| "Phone": data.get("phone", ""), | |
| "Email": data.get("email", ""), | |
| "Follower Count": data.get("follower_count", ""), | |
| "Friend Count": data.get("friend_count", ""), | |
| } | |
| except Exception as e: | |
| return {"Error": str(e)} | |
| def clean_text(text: str) -> str: | |
| return text.replace(",", " ").replace("\n", " ").strip() | |
| async def process_csv(file_path: str, scraper_func) -> List[Dict]: | |
| results = [] | |
| async with aiohttp.ClientSession() as session: | |
| with open(file_path, "r") as file: | |
| reader = csv.reader(file) | |
| next(reader) # Skip header row | |
| urls = [row[0] for row in reader] | |
| scrape_tasks = [scraper_func(url) for url in urls] | |
| results = await asyncio.gather(*scrape_tasks) | |
| with open("output.csv", "w", newline="") as file: | |
| writer = csv.DictWriter(file, fieldnames=results[0].keys()) | |
| writer.writeheader() | |
| for result in results: | |
| writer.writerow({k: clean_text(str(v)) for k, v in result.items()}) | |
| return results | |
| async def preview_csv(file_path: str, scraper_func): | |
| results = await process_csv(file_path, scraper_func) | |
| return gr.DataFrame.from_dict(results) | |
| with gr.Blocks() as demo: | |
| linkedin_inputs = gr.Textbox(label="LinkedIn Profile URLs (one per line)") | |
| with gr.Tab("LinkedIn"): | |
| linkedin_csv_file = gr.File(label="Upload CSV", type="filepath") | |
| linkedin_submit_btn = gr.Button("Scrape LinkedIn Profiles") | |
| linkedin_preview_btn = gr.Button("Preview Results") | |
| linkedin_preview_output = gr.Dataframe(label="Preview") | |
| linkedin_scrape_output = gr.Dataframe(label="Scraped Data", visible=False) | |
| facebook_inputs = gr.Textbox(label="Facebook Profile URLs (one per line)") | |
| with gr.Tab("Facebook"): | |
| facebook_csv_file = gr.File(label="Upload CSV", type="filepath") | |
| facebook_submit_btn = gr.Button("Scrape Facebook Profiles") | |
| facebook_preview_btn = gr.Button("Preview Results") | |
| facebook_preview_output = gr.Dataframe(label="Preview") | |
| facebook_scrape_output = gr.Dataframe(label="Scraped Data", visible=False) | |
| linkedin_submit_btn.click(process_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_scrape_output) | |
| linkedin_preview_btn.click(preview_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_preview_output) | |
| facebook_submit_btn.click(process_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_scrape_output) | |
| facebook_preview_btn.click(preview_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_preview_output) | |
| if __name__ == "__main__": | |
| demo.launch() |