File size: 5,946 Bytes
d5b46b1
c7bd162
 
d5b46b1
 
c7bd162
d5b46b1
d130f30
d5b46b1
 
 
 
 
 
 
 
6290c5a
c7bd162
d5b46b1
 
c7bd162
 
d5b46b1
c7bd162
d5b46b1
 
 
c7bd162
 
d5b46b1
 
 
 
 
 
 
 
 
 
 
 
 
d130f30
294970a
 
 
 
 
 
 
 
 
 
 
 
 
 
d5b46b1
32a6e9e
d5b46b1
abd564d
d5b46b1
 
 
 
 
 
 
 
 
 
 
abd564d
d5b46b1
abd564d
d5b46b1
32a6e9e
d5b46b1
 
 
 
 
 
 
 
 
 
32a6e9e
d5b46b1
d130f30
d5b46b1
 
0b6f916
d5b46b1
 
 
 
9ec997a
d5b46b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e127f06
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0b6f916
91b51b3
e127f06
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
import os
import asyncio
import aiohttp
import csv
from typing import List, Dict
import gradio as gr
import logging

logging.basicConfig(level=logging.INFO)

# Get environment variables
LINKEDIN_API_KEY = os.environ.get("LINKEDIN_API_KEY")
FACEBOOK_API_KEY = os.environ.get("FACEBOOK_API_KEY")

LINKEDIN_API_ENDPOINT = os.environ.get("LINKEDIN_API_ENDPOINT")
FACEBOOK_API_ENDPOINT = os.environ.get("FACEBOOK_API_ENDPOINT")

# Rate limiting configuration
RATE_LIMIT = 1  # Requests per second
MAX_TOKENS = 10  # Maximum tokens in the bucket

class TokenBucket:
    def __init__(self, rate, capacity):
        self.rate = rate
        self.capacity = capacity
        self.tokens = capacity
        self.last_update = asyncio.get_event_loop().time()

    async def get_token(self):
        now = asyncio.get_event_loop().time()
        elapsed_time = now - self.last_update
        self.last_update = now
        new_tokens = elapsed_time * self.rate
        self.tokens = min(self.tokens + new_tokens, self.capacity)
        if self.tokens >= 1:
            self.tokens -= 1
            return True
        return False

# Create token bucket instances for rate limiting
linkedin_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS)
facebook_token_bucket = TokenBucket(RATE_LIMIT, MAX_TOKENS)

async def rate_limited_api_call(url, headers):
    while True:
        if "linkedin" in url:
            if await linkedin_token_bucket.get_token():
                logging.info(f"Making LinkedIn API request: URL={url}, Headers={headers}")
                async with aiohttp.ClientSession() as session:
                    async with session.get(url, headers=headers) as response:
                        return await response.json()
        elif "facebook" in url:
            if await facebook_token_bucket.get_token():
                logging.info(f"Making Facebook API request: URL={url}, Headers={headers}")
                async with aiohttp.ClientSession() as session:
                    async with session.get(url, headers=headers) as response:
                        return await response.json()
        await asyncio.sleep(0.1)  # Wait before retrying

async def scrape_linkedin_profile(profile_url: str) -> Dict:
    try:
        headers = {"Authorization": f"Bearer {LINKEDIN_API_KEY}"}
        data = await rate_limited_api_call(f"{LINKEDIN_API_ENDPOINT}?url={profile_url}", headers)
        return {
            "Name": data.get("name", ""),
            "Number of Followers": data.get("num_followers", ""),
            "Number of Connections": data.get("num_connections", ""),
            "Age": data.get("age", ""),
            "Current Role": data.get("current_role", ""),
            "Sex": data.get("sex", ""),
            "Place of Living": data.get("place_of_living", ""),
        }
    except Exception as e:
        return {"Error": str(e)}

async def scrape_facebook_profile(profile_url: str) -> Dict:
    try:
        headers = {"Authorization": f"Bearer {FACEBOOK_API_KEY}"}
        data = await rate_limited_api_call(f"{FACEBOOK_API_ENDPOINT}?url={profile_url}", headers)
        return {
            "Full Name": data.get("full_name", ""),
            "Location": data.get("location", ""),
            "Phone": data.get("phone", ""),
            "Email": data.get("email", ""),
            "Follower Count": data.get("follower_count", ""),
            "Friend Count": data.get("friend_count", ""),
        }
    except Exception as e:
        return {"Error": str(e)}

def clean_text(text: str) -> str:
    return text.replace(",", " ").replace("\n", " ").strip()

async def process_csv(file_path: str, scraper_func) -> List[Dict]:
    results = []
    async with aiohttp.ClientSession() as session:
        with open(file_path, "r") as file:
            reader = csv.reader(file)
            next(reader)  # Skip header row
            urls = [row[0] for row in reader]

        scrape_tasks = [scraper_func(url) for url in urls]
        results = await asyncio.gather(*scrape_tasks)

    with open("output.csv", "w", newline="") as file:
        writer = csv.DictWriter(file, fieldnames=results[0].keys())
        writer.writeheader()
        for result in results:
            writer.writerow({k: clean_text(str(v)) for k, v in result.items()})

    return results

async def preview_csv(file_path: str, scraper_func):
    results = await process_csv(file_path, scraper_func)
    return gr.DataFrame.from_dict(results)

with gr.Blocks() as demo:
    linkedin_inputs = gr.Textbox(label="LinkedIn Profile URLs (one per line)")
    with gr.Tab("LinkedIn"):
        linkedin_csv_file = gr.File(label="Upload CSV", type="filepath")
        linkedin_submit_btn = gr.Button("Scrape LinkedIn Profiles")
        linkedin_preview_btn = gr.Button("Preview Results")
        linkedin_preview_output = gr.Dataframe(label="Preview")
        linkedin_scrape_output = gr.Dataframe(label="Scraped Data", visible=False)

    facebook_inputs = gr.Textbox(label="Facebook Profile URLs (one per line)")
    with gr.Tab("Facebook"):
        facebook_csv_file = gr.File(label="Upload CSV", type="filepath")
        facebook_submit_btn = gr.Button("Scrape Facebook Profiles")
        facebook_preview_btn = gr.Button("Preview Results")
        facebook_preview_output = gr.Dataframe(label="Preview")
        facebook_scrape_output = gr.Dataframe(label="Scraped Data", visible=False)

    linkedin_submit_btn.click(process_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_scrape_output)
    linkedin_preview_btn.click(preview_csv, inputs=[linkedin_csv_file, scrape_linkedin_profile], outputs=linkedin_preview_output)

    facebook_submit_btn.click(process_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_scrape_output)
    facebook_preview_btn.click(preview_csv, inputs=[facebook_csv_file, scrape_facebook_profile], outputs=facebook_preview_output)

if __name__ == "__main__":
    demo.launch()