jayanth-2005 commited on
Commit
2b19d86
Β·
verified Β·
1 Parent(s): 6a3e88a

Update src/streamlit_app.py

Browse files
Files changed (1) hide show
  1. src/streamlit_app.py +88 -56
src/streamlit_app.py CHANGED
@@ -4,94 +4,125 @@ from bs4 import BeautifulSoup
4
  import pandas as pd
5
  import re
6
  import urllib3
 
7
 
8
  urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
9
 
10
- st.set_page_config(page_title="Faculty Scraper", layout="wide")
11
-
12
- st.title("πŸŽ“ Universal Faculty Scraper (API + Static)")
13
- st.write("Handles both normal and JavaScript-based faculty pages πŸš€")
14
 
15
  EMAIL_REGEX = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
16
  PHONE_REGEX = r"\+?\d[\d\s\-]{8,15}"
17
 
18
- # -------- NAME FROM EMAIL --------
19
- def name_from_email(email):
20
- name = email.split("@")[0]
21
- name = name.replace(".", " ").replace("_", " ")
22
- return " ".join(w.capitalize() for w in name.split())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
 
24
- # -------- STATIC SCRAPER --------
25
- def scrape_static(url):
26
  headers = {"User-Agent": "Mozilla/5.0"}
27
 
28
  try:
29
  res = requests.get(url, headers=headers, verify=False, timeout=10)
30
  except:
31
- return pd.DataFrame()
32
 
33
  soup = BeautifulSoup(res.text, "html.parser")
34
  text = soup.get_text(" ", strip=True)
35
 
36
- emails = list(set(re.findall(EMAIL_REGEX, text)))
37
- phones = list(set(re.findall(PHONE_REGEX, text)))
38
 
39
  data = []
40
- for i, email in enumerate(emails):
41
- name = name_from_email(email)
42
- phone = phones[i] if i < len(phones) else "Not Available"
43
- data.append([name, email, phone])
44
 
45
- return pd.DataFrame(data, columns=["Name", "Email", "Phone"])
 
 
46
 
47
- # -------- API SCRAPER (RKNEC FIX) --------
48
- def scrape_api(url):
49
- try:
50
- base = url.split("/computer")[0]
51
 
52
- api_url = base + "/wp-json/wp/v2/pages?search=faculty"
 
53
 
54
- res = requests.get(api_url, timeout=10)
55
 
56
- if res.status_code != 200:
57
- return pd.DataFrame()
 
 
58
 
59
- data_json = res.json()
 
60
 
61
- all_text = ""
62
 
63
- for item in data_json:
64
- if "content" in item and "rendered" in item["content"]:
65
- html = item["content"]["rendered"]
66
- soup = BeautifulSoup(html, "html.parser")
67
- all_text += soup.get_text(" ", strip=True) + " "
68
 
69
- emails = list(set(re.findall(EMAIL_REGEX, all_text)))
70
- phones = list(set(re.findall(PHONE_REGEX, all_text)))
 
 
71
 
72
- data = []
73
 
74
- for i, email in enumerate(emails):
75
- name = name_from_email(email)
76
- phone = phones[i] if i < len(phones) else "Not Available"
77
- data.append([name, email, phone])
78
 
79
- return pd.DataFrame(data, columns=["Name", "Email", "Phone"])
 
80
 
81
- except:
82
- return pd.DataFrame()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
83
 
84
- # -------- MAIN --------
85
- def scrape(url):
86
- # 1️⃣ Try static
87
- df = scrape_static(url)
88
 
89
- # 2️⃣ If weak β†’ try API
90
- if df.empty or len(df) < 5:
91
- st.info("πŸ”„ Trying API-based extraction...")
92
- df = scrape_api(url)
93
 
94
- return df
95
 
96
  # -------- UI --------
97
  url = st.text_input("πŸ”— Enter Faculty Page URL")
@@ -100,8 +131,9 @@ if st.button("πŸš€ Scrape Data"):
100
  if not url:
101
  st.warning("Enter a URL")
102
  else:
103
- with st.spinner("Scraping..."):
104
- df = scrape(url)
 
105
 
106
  if not df.empty:
107
  st.success("βœ… Data Extracted Successfully")
@@ -116,4 +148,4 @@ if st.button("πŸš€ Scrape Data"):
116
  "text/csv"
117
  )
118
  else:
119
- st.error("❌ Could not extract data (site may block scraping)")
 
4
  import pandas as pd
5
  import re
6
  import urllib3
7
+ from urllib.parse import urljoin
8
 
9
  urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
10
 
11
+ st.title("πŸŽ“ AI Faculty Scraper (Pro Version)")
 
 
 
12
 
13
  EMAIL_REGEX = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
14
  PHONE_REGEX = r"\+?\d[\d\s\-]{8,15}"
15
 
16
+ DESIGNATIONS = [
17
+ "professor", "assistant professor", "associate professor",
18
+ "hod", "head", "lecturer", "faculty"
19
+ ]
20
+
21
+ NAME_TITLES = ["dr", "mr", "ms", "prof"]
22
+
23
+ # -------- SMART NAME DETECTION --------
24
+ def extract_name(text):
25
+ words = text.split()
26
+
27
+ for i in range(len(words)):
28
+ candidate = " ".join(words[i:i+4])
29
+
30
+ if any(t in candidate.lower() for t in NAME_TITLES):
31
+ return candidate.strip()
32
+
33
+ return None
34
+
35
+
36
+ # -------- DESIGNATION DETECTION --------
37
+ def extract_designation(text):
38
+ for d in DESIGNATIONS:
39
+ if d in text.lower():
40
+ return d.title()
41
+ return "Not Available"
42
+
43
 
44
+ # -------- EXTRACT FROM PAGE --------
45
+ def extract_from_page(url):
46
  headers = {"User-Agent": "Mozilla/5.0"}
47
 
48
  try:
49
  res = requests.get(url, headers=headers, verify=False, timeout=10)
50
  except:
51
+ return [], []
52
 
53
  soup = BeautifulSoup(res.text, "html.parser")
54
  text = soup.get_text(" ", strip=True)
55
 
56
+ emails = re.findall(EMAIL_REGEX, text)
57
+ phones = re.findall(PHONE_REGEX, text)
58
 
59
  data = []
 
 
 
 
60
 
61
+ for email in set(emails):
62
+ idx = text.find(email)
63
+ context = text[max(0, idx-150):idx+50]
64
 
65
+ name = extract_name(context)
66
+ designation = extract_designation(context)
 
 
67
 
68
+ phone_match = re.search(PHONE_REGEX, context)
69
+ phone = phone_match.group() if phone_match else "Not Available"
70
 
71
+ data.append([name, email, phone, designation])
72
 
73
+ # -------- FIND MORE LINKS --------
74
+ links = []
75
+ for a in soup.find_all("a", href=True):
76
+ link = urljoin(url, a["href"])
77
 
78
+ if "faculty" in link.lower() or "profile" in link.lower():
79
+ links.append(link)
80
 
81
+ return data, list(set(links))
82
 
 
 
 
 
 
83
 
84
+ # -------- MAIN CRAWLER --------
85
+ def crawl_site(start_url, max_pages=5):
86
+ visited = set()
87
+ to_visit = [start_url]
88
 
89
+ all_data = []
90
 
91
+ while to_visit and len(visited) < max_pages:
92
+ url = to_visit.pop(0)
 
 
93
 
94
+ if url in visited:
95
+ continue
96
 
97
+ visited.add(url)
98
+
99
+ data, links = extract_from_page(url)
100
+ all_data.extend(data)
101
+
102
+ for link in links:
103
+ if link not in visited:
104
+ to_visit.append(link)
105
+
106
+ return pd.DataFrame(all_data, columns=["Name", "Email", "Phone", "Designation"])
107
+
108
+
109
+ # -------- CLEAN DATA --------
110
+ def clean_data(df):
111
+ cleaned = []
112
+
113
+ for email, group in df.groupby("Email"):
114
+ names = [n for n in group["Name"] if n]
115
+ phones = group["Phone"].tolist()
116
+ designations = group["Designation"].tolist()
117
+
118
+ best_name = max(names, key=len) if names else "Not Available"
119
+ best_phone = next((p for p in phones if p != "Not Available"), "Not Available")
120
+ best_designation = next((d for d in designations if d != "Not Available"), "Not Available")
121
 
122
+ cleaned.append([best_name, email, best_phone, best_designation])
 
 
 
123
 
124
+ return pd.DataFrame(cleaned, columns=["Name", "Email", "Phone", "Designation"])
 
 
 
125
 
 
126
 
127
  # -------- UI --------
128
  url = st.text_input("πŸ”— Enter Faculty Page URL")
 
131
  if not url:
132
  st.warning("Enter a URL")
133
  else:
134
+ with st.spinner("Crawling and extracting..."):
135
+ df = crawl_site(url)
136
+ df = clean_data(df)
137
 
138
  if not df.empty:
139
  st.success("βœ… Data Extracted Successfully")
 
148
  "text/csv"
149
  )
150
  else:
151
+ st.error("No data found")