JaySMTA commited on
Commit
32a6e9e
·
verified ·
1 Parent(s): a2e0956

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +123 -230
app.py CHANGED
@@ -1,6 +1,5 @@
1
  import os
2
  import csv
3
- import socket
4
  import requests
5
  import gradio as gr
6
  import logging
@@ -9,114 +8,112 @@ import random
9
  import json
10
  from concurrent.futures import ThreadPoolExecutor, as_completed
11
  from ratelimit import limits, sleep_and_retry
12
- from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
13
- from requests.exceptions import RequestException, ConnectionError
14
-
15
 
 
16
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
17
 
18
-
19
- def get_secret(secret_name):
20
-
21
- return "your_api_key_here"
22
-
23
 
24
  @sleep_and_retry
25
- @limits(calls=5, period=60)
26
- def rate_limited_api_call(url, headers, data):
27
- return requests.post(url, json=data, headers=headers)
28
-
29
 
30
  @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
31
- def resilient_api_call(url, headers, data):
32
  try:
33
- response = rate_limited_api_call(url, headers, data)
34
  response.raise_for_status()
35
  return response.json()
36
  except RequestException as e:
37
- logging.error(f"API call failed: {str(e)}")
38
  raise
39
 
40
-
41
- def human_like_delay():
42
- delay = random.uniform(1, 3)
43
- logging.info(f"Waiting for {delay:.2f} seconds")
44
- time.sleep(delay)
45
-
46
-
47
  def scrape_linkedin_profile(url):
 
 
 
 
 
48
  api_url = "https://zylalabs.com/api/1511/linkedin+profile+scraper+api/1250/get+data"
49
- api_key = get_secret('ZYLALABS_API_KEY')
50
-
51
  headers = {
52
- 'Authorization': f'Bearer {api_key}'
 
53
  }
54
-
55
- params = {
56
  "url": url
57
  }
58
-
59
  try:
60
- response = requests.get(api_url, headers=headers, params=params)
61
- response.raise_for_status()
62
- result = response.json()
63
  if result.get('message') == 'ok':
64
- return result['data']
 
65
  else:
66
- logging.error(f"API error for profile {url}: {result.get('message')}")
67
  return None
68
  except Exception as e:
69
- logging.error(f"Error scraping profile {url}: {str(e)}")
70
  return None
71
 
 
 
 
 
 
72
 
73
- def scrape_linkedin_followers(profile_url, progress=gr.Progress()):
74
- api_key = get_secret('ZYLALABS_API_KEY')
75
- api_url = "https://api.zylalabs.com/api/1099/linkedin-profile-follower-scraper-api/1248/linkedin-profile-followers"
76
-
77
  headers = {
78
  'Authorization': f'Bearer {api_key}',
79
  'Content-Type': 'application/json'
80
  }
81
-
82
- all_followers = []
83
- page = 1
84
- total_followers = None
85
-
86
- while True:
87
- data = {
88
- "url": profile_url,
89
- "page": page
90
- }
91
-
92
- try:
93
- result = resilient_api_call(api_url, headers, data)
94
-
95
- if 'followers' in result:
96
- followers = result['followers']
97
- all_followers.extend([follower['profileUrl'] for follower in followers])
98
-
99
- if total_followers is None:
100
- total_followers = result.get('totalFollowers', len(followers))
101
- progress(0, desc=f"Scraping {total_followers} followers")
102
-
103
- progress(len(all_followers) / total_followers, desc=f"Scraped {len(all_followers)}/{total_followers}")
104
-
105
- if len(followers) < 100:
106
- break
107
- page += 1
108
- else:
109
- break
110
-
111
- human_like_delay()
112
- except Exception as e:
113
- logging.error(f"Error scraping followers for {profile_url}: {str(e)}")
114
- break
115
-
116
- return all_followers
117
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
 
119
- def process_linkedin_csv(csv_file):
120
  try:
121
  with open(csv_file.name, 'r') as file:
122
  reader = csv.reader(file)
@@ -124,7 +121,7 @@ def process_linkedin_csv(csv_file):
124
 
125
  results = []
126
  with ThreadPoolExecutor(max_workers=5) as executor:
127
- future_to_url = {executor.submit(scrape_linkedin_profile, url): url for url in profile_urls}
128
  for future in as_completed(future_to_url):
129
  url = future_to_url[future]
130
  try:
@@ -139,86 +136,7 @@ def process_linkedin_csv(csv_file):
139
  logging.error(f"Error processing CSV file: {str(e)}")
140
  return f"Error processing CSV file: {str(e)}"
141
 
142
-
143
- def process_linkedin_followers_csv(csv_file, progress=gr.Progress()):
144
- if not check_api_availability():
145
- return "Error: The LinkedIn API is not available. Please check your internet connection or try again later."
146
-
147
- try:
148
- with open(csv_file.name, 'r') as file:
149
- reader = csv.reader(file)
150
- profile_urls = [row[0] for row in reader if row]
151
-
152
- if len(profile_urls) != 1:
153
- return "Error: CSV file should contain exactly one LinkedIn profile URL."
154
-
155
- profile_url = profile_urls[0]
156
- follower_links = scrape_linkedin_followers(profile_url, progress)
157
-
158
- if follower_links:
159
- return "\n".join(follower_links)
160
- else:
161
- return "No followers found or an error occurred while scraping."
162
- except Exception as e:
163
- logging.error(f"Error processing CSV file: {str(e)}")
164
- return f"Error processing CSV file: {str(e)}"
165
-
166
-
167
- def scrape_linkedin_followers(profile_url, progress=gr.Progress()):
168
- if not check_api_availability():
169
- logging.error("API is not available. Please check your internet connection or the API status.")
170
- return None
171
-
172
- api_key = get_secret('ZYLALABS_API_KEY')
173
- api_url = "https://api.zylalabs.com/api/1099/linkedin-profile-follower-scraper-api/1248/linkedin-profile-followers"
174
-
175
- headers = {
176
- 'Authorization': f'Bearer {api_key}',
177
- 'Content-Type': 'application/json'
178
- }
179
-
180
- all_followers = []
181
- page = 1
182
- total_followers = None
183
-
184
- while True:
185
- data = {
186
- "url": profile_url,
187
- "page": page
188
- }
189
-
190
- try:
191
- result = resilient_api_call(api_url, headers, data)
192
-
193
- if 'followers' in result:
194
- followers = result['followers']
195
- all_followers.extend([follower['profileUrl'] for follower in followers])
196
-
197
- if total_followers is None:
198
- total_followers = result.get('totalFollowers', len(followers))
199
- progress(0, desc=f"Scraping {total_followers} followers")
200
-
201
- progress(len(all_followers) / total_followers, desc=f"Scraped {len(all_followers)}/{total_followers}")
202
-
203
- if len(followers) < 100: # Assuming 100 is the batch size
204
- break
205
- page += 1
206
- else:
207
- break
208
-
209
- human_like_delay()
210
- except Exception as e:
211
- logging.error(f"Error scraping followers for {profile_url}: {str(e)}")
212
- break
213
-
214
- return all_followers
215
-
216
- def process_facebook_csv(csv_file):
217
-
218
- return "Facebook scraping not implemented yet."
219
-
220
-
221
- def preview_linkedin_csv(csv_file):
222
  try:
223
  with open(csv_file.name, 'r') as file:
224
  reader = csv.reader(file)
@@ -228,7 +146,7 @@ def preview_linkedin_csv(csv_file):
228
  return "No URLs found in the CSV file."
229
 
230
  preview_url = profile_urls[0]
231
- preview_data = scrape_linkedin_profile(preview_url)
232
 
233
  if preview_data:
234
  return json.dumps(preview_data, indent=2)
@@ -238,83 +156,59 @@ def preview_linkedin_csv(csv_file):
238
  logging.error(f"Error previewing CSV file: {str(e)}")
239
  return f"Error previewing CSV file: {str(e)}"
240
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
 
242
- def preview_linkedin_followers_csv(csv_file):
243
- try:
244
- with open(csv_file.name, 'r') as file:
245
- reader = csv.reader(file)
246
- profile_urls = [row[0] for row in reader if row]
247
-
248
- if len(profile_urls) != 1:
249
- return "Error: CSV file should contain exactly one LinkedIn profile URL."
250
-
251
- profile_url = profile_urls[0]
252
- preview_followers = scrape_linkedin_followers(profile_url, progress=None)[:5] # Get first 5 followers
253
-
254
- if preview_followers:
255
- return "\n".join(preview_followers)
256
- else:
257
- return "No followers found or an error occurred while scraping."
258
- except Exception as e:
259
- logging.error(f"Error previewing CSV file: {str(e)}")
260
- return f"Error previewing CSV file: {str(e)}"
261
-
262
- # Create LinkedIn interface
263
- def create_linkedin_interface():
264
- linkedin_input = gr.File(label="Upload CSV file with LinkedIn profile URLs")
265
- linkedin_preview = gr.Textbox(label="Preview (First Profile)", max_lines=10)
266
- linkedin_output = gr.Textbox(label="Scraped LinkedIn Data")
267
- linkedin_preview_button = gr.Button("Preview")
268
- linkedin_scrape_button = gr.Button("Scrape LinkedIn Profiles")
269
-
270
- def check_and_preview(csv_file):
271
- if not check_api_availability():
272
- return "Error: The Linkedin API is not available. Please check your connection or try again later."
273
-
274
- linkedin_preview_button.click(check_and_preview, inputs=linkedin_input, outputs=linkedin_preview)
275
- linkedin_scrape_button.click(process_linkedin_csv, inputs=linkedin_input, outputs=linkedin_output)
276
-
277
- return linkedin_input, linkedin_preview, linkedin_output
278
-
279
- # Create Facebook interface
280
- def create_facebook_interface():
281
- facebook_input = gr.File(label="Upload CSV file with Facebook profile URLs")
282
- facebook_preview = gr.Textbox(label="Preview (First Profile)", max_lines=10)
283
- facebook_output = gr.Textbox(label="Scraped Facebook Data")
284
- facebook_preview_button = gr.Button("Preview")
285
- facebook_scrape_button = gr.Button("Scrape Facebook Profiles")
286
-
287
- facebook_preview_button.click(lambda x: "Facebook preview not implemented yet.", inputs=facebook_input, outputs=facebook_preview)
288
- facebook_scrape_button.click(process_facebook_csv, inputs=facebook_input, outputs=facebook_output)
289
-
290
- return facebook_input, facebook_preview, facebook_output
291
-
292
- # Create LinkedIn Follower interface
293
- def create_linkedin_follower_interface():
294
- followers_input = gr.File(label="Upload CSV file with a single LinkedIn profile URL")
295
- followers_preview = gr.Textbox(label="Preview (First 5 Followers)", max_lines=5)
296
- followers_output = gr.Textbox(label="Follower Links")
297
- followers_preview_button = gr.Button("Preview")
298
- followers_scrape_button = gr.Button("Scrape Followers")
299
-
300
- followers_preview_button.click(preview_linkedin_followers_csv, inputs=followers_input, outputs=followers_preview)
301
- followers_scrape_button.click(process_linkedin_followers_csv, inputs=followers_input, outputs=followers_output)
302
-
303
- return followers_input, followers_preview, followers_output
304
-
305
- # Create the main application
306
  def create_app():
307
  with gr.Blocks() as demo:
308
  gr.Markdown("# Multi-functional Data Processing Application")
309
 
310
  with gr.Tab("LinkedIn Scraper"):
311
- linkedin_input, linkedin_preview, linkedin_output = create_linkedin_interface()
312
 
313
  with gr.Tab("Facebook Scraper"):
314
- facebook_input, facebook_preview, facebook_output = create_facebook_interface()
315
-
316
- with gr.Tab("LinkedIn Follower Scraper"):
317
- followers_input, followers_preview, followers_output = create_linkedin_follower_interface()
318
 
319
  return demo
320
 
@@ -322,10 +216,9 @@ def create_app():
322
  app = create_app()
323
 
324
  if __name__ == "__main__":
325
- # Check if we're running on Hugging Face Spaces
326
  if "SPACE_ID" in os.environ:
327
-
328
  app.launch()
329
  else:
330
- # We're running locally, use share
331
  app.launch(share=True)
 
1
  import os
2
  import csv
 
3
  import requests
4
  import gradio as gr
5
  import logging
 
8
  import json
9
  from concurrent.futures import ThreadPoolExecutor, as_completed
10
  from ratelimit import limits, sleep_and_retry
11
+ from tenacity import retry, stop_after_attempt, wait_exponential
12
+ from requests.exceptions import RequestException
 
13
 
14
+ # Set up logging
15
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
16
 
17
+ def get_zylalabs_api_key():
18
+ return os.environ.get('ZYLALABS_API_KEY')
 
 
 
19
 
20
  @sleep_and_retry
21
+ @limits(calls=5, period=60) # Adjust these values based on Zylalabs rate limits
22
+ def zylalabs_rate_limited_api_call(url, headers, data):
23
+ return requests.post(url, headers=headers, json=data)
 
24
 
25
  @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
26
+ def resilient_zylalabs_api_call(url, headers, data):
27
  try:
28
+ response = zylalabs_rate_limited_api_call(url, headers, data)
29
  response.raise_for_status()
30
  return response.json()
31
  except RequestException as e:
32
+ logging.error(f"Zylalabs API call failed: {str(e)}")
33
  raise
34
 
 
 
 
 
 
 
 
35
  def scrape_linkedin_profile(url):
36
+ api_key = get_zylalabs_api_key()
37
+ if not api_key:
38
+ logging.error("ZYLALABS_API_KEY not found in environment variables")
39
+ return None
40
+
41
  api_url = "https://zylalabs.com/api/1511/linkedin+profile+scraper+api/1250/get+data"
 
 
42
  headers = {
43
+ 'Authorization': f'Bearer {api_key}',
44
+ 'Content-Type': 'application/json'
45
  }
46
+ data = {
 
47
  "url": url
48
  }
49
+
50
  try:
51
+ result = resilient_zylalabs_api_call(api_url, headers, data)
 
 
52
  if result.get('message') == 'ok':
53
+ logging.info(f"LinkedIn profile data: {json.dumps(result.get('data'), indent=2)}")
54
+ return result.get('data')
55
  else:
56
+ logging.error(f"API error for LinkedIn profile {url}: {result.get('message')}")
57
  return None
58
  except Exception as e:
59
+ logging.error(f"Error scraping LinkedIn profile {url}: {str(e)}")
60
  return None
61
 
62
+ def scrape_facebook_profile(url):
63
+ api_key = get_zylalabs_api_key()
64
+ if not api_key:
65
+ logging.error("ZYLALABS_API_KEY not found in environment variables")
66
+ return None
67
 
68
+ api_url = "https://zylalabs.com/api/4512/facebook+scraper+-+fetcher+pro+api/5534/profile"
 
 
 
69
  headers = {
70
  'Authorization': f'Bearer {api_key}',
71
  'Content-Type': 'application/json'
72
  }
73
+ data = {
74
+ "profile_url": url
75
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
 
77
+ try:
78
+ result = resilient_zylalabs_api_call(api_url, headers, data)
79
+ if 'error' not in result:
80
+ profile_data = {
81
+ "Name": result.get('name', ''),
82
+ "Profile Link": url,
83
+ "Bio": result.get('bio', ''),
84
+ "About Info": result.get('about', ''),
85
+ "Websites": result.get('websites', []),
86
+ "Social Media Links": result.get('social_media_links', []),
87
+ "Emails": result.get('emails', []),
88
+ "Current Location": result.get('current_location', ''),
89
+ "Age": result.get('age', ''),
90
+ "Gender": result.get('gender', ''),
91
+ "Profession": result.get('profession', ''),
92
+ "Friends Count": result.get('friends_count', ''),
93
+ "Followers Count": result.get('followers_count', ''),
94
+ "Following Count": result.get('following_count', ''),
95
+ }
96
+
97
+ # Fetch the latest 45 posts
98
+ posts = result.get('posts', [])[:45]
99
+ profile_data['Latest Posts'] = [{
100
+ "Content": post.get('content', ''),
101
+ "Has Image": 'Yes' if post.get('image_url') else 'No',
102
+ "Likes Count": post.get('likes_count', ''),
103
+ "Comments Count": post.get('comments_count', ''),
104
+ "Shares Count": post.get('shares_count', ''),
105
+ "Date": post.get('date', '')
106
+ } for post in posts]
107
+
108
+ return profile_data
109
+ else:
110
+ logging.error(f"API error for Facebook profile {url}: {result.get('error')}")
111
+ return None
112
+ except Exception as e:
113
+ logging.error(f"Error scraping Facebook profile {url}: {str(e)}")
114
+ return None
115
 
116
+ def process_csv(csv_file, scrape_function):
117
  try:
118
  with open(csv_file.name, 'r') as file:
119
  reader = csv.reader(file)
 
121
 
122
  results = []
123
  with ThreadPoolExecutor(max_workers=5) as executor:
124
+ future_to_url = {executor.submit(scrape_function, url): url for url in profile_urls}
125
  for future in as_completed(future_to_url):
126
  url = future_to_url[future]
127
  try:
 
136
  logging.error(f"Error processing CSV file: {str(e)}")
137
  return f"Error processing CSV file: {str(e)}"
138
 
139
+ def preview_csv(csv_file, scrape_function):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
140
  try:
141
  with open(csv_file.name, 'r') as file:
142
  reader = csv.reader(file)
 
146
  return "No URLs found in the CSV file."
147
 
148
  preview_url = profile_urls[0]
149
+ preview_data = scrape_function(preview_url)
150
 
151
  if preview_data:
152
  return json.dumps(preview_data, indent=2)
 
156
  logging.error(f"Error previewing CSV file: {str(e)}")
157
  return f"Error previewing CSV file: {str(e)}"
158
 
159
+ def create_interface(name, scrape_function):
160
+ input_file = gr.File(label=f"Upload CSV file with {name} profile URLs")
161
+ preview = gr.Textbox(label="Preview (First Profile)", max_lines=10)
162
+ output = gr.Textbox(label=f"Scraped {name} Data")
163
+ preview_button = gr.Button("Preview")
164
+ scrape_button = gr.Button(f"Scrape {name} Profiles")
165
+ download = gr.File(label="Download Scraped Data as CSV")
166
+
167
+ def scrape_and_download(csv_file):
168
+ result = process_csv(csv_file, scrape_function)
169
+ if result.startswith("Error"):
170
+ return None
171
+ data = json.loads(result)
172
+ output_file = f"{name.lower()}_scraped_data.csv"
173
+ with open(output_file, 'w', newline='', encoding='utf-8') as f:
174
+ if data:
175
+ fieldnames = list(data[0].keys())
176
+ if 'Latest Posts' in fieldnames:
177
+ fieldnames.remove('Latest Posts')
178
+ for i in range(45):
179
+ fieldnames.extend([f"Post {i+1} Content", f"Post {i+1} Has Image",
180
+ f"Post {i+1} Likes Count", f"Post {i+1} Comments Count",
181
+ f"Post {i+1} Shares Count", f"Post {i+1} Date"])
182
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
183
+ writer.writeheader()
184
+ for row in data:
185
+ flat_row = {k: v for k, v in row.items() if k != 'Latest Posts'}
186
+ if 'Latest Posts' in row:
187
+ for i, post in enumerate(row['Latest Posts']):
188
+ flat_row[f"Post {i+1} Content"] = post['Content']
189
+ flat_row[f"Post {i+1} Has Image"] = post['Has Image']
190
+ flat_row[f"Post {i+1} Likes Count"] = post['Likes Count']
191
+ flat_row[f"Post {i+1} Comments Count"] = post['Comments Count']
192
+ flat_row[f"Post {i+1} Shares Count"] = post['Shares Count']
193
+ flat_row[f"Post {i+1} Date"] = post['Date']
194
+ writer.writerow(flat_row)
195
+ return output_file
196
+
197
+ preview_button.click(lambda x: preview_csv(x, scrape_function), inputs=input_file, outputs=preview)
198
+ scrape_button.click(lambda x: process_csv(x, scrape_function), inputs=input_file, outputs=output)
199
+ scrape_button.click(scrape_and_download, inputs=input_file, outputs=download)
200
+
201
+ return input_file, preview, output, download
202
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
203
  def create_app():
204
  with gr.Blocks() as demo:
205
  gr.Markdown("# Multi-functional Data Processing Application")
206
 
207
  with gr.Tab("LinkedIn Scraper"):
208
+ create_interface("LinkedIn", scrape_linkedin_profile)
209
 
210
  with gr.Tab("Facebook Scraper"):
211
+ create_interface("Facebook", scrape_facebook_profile)
 
 
 
212
 
213
  return demo
214
 
 
216
  app = create_app()
217
 
218
  if __name__ == "__main__":
 
219
  if "SPACE_ID" in os.environ:
220
+ # We're on Hugging Face Spaces
221
  app.launch()
222
  else:
223
+ # We're running locally
224
  app.launch(share=True)