Spaces:
Runtime error
Runtime error
Update src/streamlit_app.py
Browse files- src/streamlit_app.py +88 -56
src/streamlit_app.py
CHANGED
|
@@ -4,94 +4,125 @@ from bs4 import BeautifulSoup
|
|
| 4 |
import pandas as pd
|
| 5 |
import re
|
| 6 |
import urllib3
|
|
|
|
| 7 |
|
| 8 |
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
| 9 |
|
| 10 |
-
st.
|
| 11 |
-
|
| 12 |
-
st.title("π Universal Faculty Scraper (API + Static)")
|
| 13 |
-
st.write("Handles both normal and JavaScript-based faculty pages π")
|
| 14 |
|
| 15 |
EMAIL_REGEX = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
|
| 16 |
PHONE_REGEX = r"\+?\d[\d\s\-]{8,15}"
|
| 17 |
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
-
# --------
|
| 25 |
-
def
|
| 26 |
headers = {"User-Agent": "Mozilla/5.0"}
|
| 27 |
|
| 28 |
try:
|
| 29 |
res = requests.get(url, headers=headers, verify=False, timeout=10)
|
| 30 |
except:
|
| 31 |
-
return
|
| 32 |
|
| 33 |
soup = BeautifulSoup(res.text, "html.parser")
|
| 34 |
text = soup.get_text(" ", strip=True)
|
| 35 |
|
| 36 |
-
emails =
|
| 37 |
-
phones =
|
| 38 |
|
| 39 |
data = []
|
| 40 |
-
for i, email in enumerate(emails):
|
| 41 |
-
name = name_from_email(email)
|
| 42 |
-
phone = phones[i] if i < len(phones) else "Not Available"
|
| 43 |
-
data.append([name, email, phone])
|
| 44 |
|
| 45 |
-
|
|
|
|
|
|
|
| 46 |
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
try:
|
| 50 |
-
base = url.split("/computer")[0]
|
| 51 |
|
| 52 |
-
|
|
|
|
| 53 |
|
| 54 |
-
|
| 55 |
|
| 56 |
-
|
| 57 |
-
|
|
|
|
|
|
|
| 58 |
|
| 59 |
-
|
|
|
|
| 60 |
|
| 61 |
-
|
| 62 |
|
| 63 |
-
for item in data_json:
|
| 64 |
-
if "content" in item and "rendered" in item["content"]:
|
| 65 |
-
html = item["content"]["rendered"]
|
| 66 |
-
soup = BeautifulSoup(html, "html.parser")
|
| 67 |
-
all_text += soup.get_text(" ", strip=True) + " "
|
| 68 |
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
| 71 |
|
| 72 |
-
|
| 73 |
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
phone = phones[i] if i < len(phones) else "Not Available"
|
| 77 |
-
data.append([name, email, phone])
|
| 78 |
|
| 79 |
-
|
|
|
|
| 80 |
|
| 81 |
-
|
| 82 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
-
|
| 85 |
-
def scrape(url):
|
| 86 |
-
# 1οΈβ£ Try static
|
| 87 |
-
df = scrape_static(url)
|
| 88 |
|
| 89 |
-
|
| 90 |
-
if df.empty or len(df) < 5:
|
| 91 |
-
st.info("π Trying API-based extraction...")
|
| 92 |
-
df = scrape_api(url)
|
| 93 |
|
| 94 |
-
return df
|
| 95 |
|
| 96 |
# -------- UI --------
|
| 97 |
url = st.text_input("π Enter Faculty Page URL")
|
|
@@ -100,8 +131,9 @@ if st.button("π Scrape Data"):
|
|
| 100 |
if not url:
|
| 101 |
st.warning("Enter a URL")
|
| 102 |
else:
|
| 103 |
-
with st.spinner("
|
| 104 |
-
df =
|
|
|
|
| 105 |
|
| 106 |
if not df.empty:
|
| 107 |
st.success("β
Data Extracted Successfully")
|
|
@@ -116,4 +148,4 @@ if st.button("π Scrape Data"):
|
|
| 116 |
"text/csv"
|
| 117 |
)
|
| 118 |
else:
|
| 119 |
-
st.error("
|
|
|
|
| 4 |
import pandas as pd
|
| 5 |
import re
|
| 6 |
import urllib3
|
| 7 |
+
from urllib.parse import urljoin
|
| 8 |
|
| 9 |
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
| 10 |
|
| 11 |
+
st.title("π AI Faculty Scraper (Pro Version)")
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
EMAIL_REGEX = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
|
| 14 |
PHONE_REGEX = r"\+?\d[\d\s\-]{8,15}"
|
| 15 |
|
| 16 |
+
DESIGNATIONS = [
|
| 17 |
+
"professor", "assistant professor", "associate professor",
|
| 18 |
+
"hod", "head", "lecturer", "faculty"
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
NAME_TITLES = ["dr", "mr", "ms", "prof"]
|
| 22 |
+
|
| 23 |
+
# -------- SMART NAME DETECTION --------
|
| 24 |
+
def extract_name(text):
|
| 25 |
+
words = text.split()
|
| 26 |
+
|
| 27 |
+
for i in range(len(words)):
|
| 28 |
+
candidate = " ".join(words[i:i+4])
|
| 29 |
+
|
| 30 |
+
if any(t in candidate.lower() for t in NAME_TITLES):
|
| 31 |
+
return candidate.strip()
|
| 32 |
+
|
| 33 |
+
return None
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
# -------- DESIGNATION DETECTION --------
|
| 37 |
+
def extract_designation(text):
|
| 38 |
+
for d in DESIGNATIONS:
|
| 39 |
+
if d in text.lower():
|
| 40 |
+
return d.title()
|
| 41 |
+
return "Not Available"
|
| 42 |
+
|
| 43 |
|
| 44 |
+
# -------- EXTRACT FROM PAGE --------
|
| 45 |
+
def extract_from_page(url):
|
| 46 |
headers = {"User-Agent": "Mozilla/5.0"}
|
| 47 |
|
| 48 |
try:
|
| 49 |
res = requests.get(url, headers=headers, verify=False, timeout=10)
|
| 50 |
except:
|
| 51 |
+
return [], []
|
| 52 |
|
| 53 |
soup = BeautifulSoup(res.text, "html.parser")
|
| 54 |
text = soup.get_text(" ", strip=True)
|
| 55 |
|
| 56 |
+
emails = re.findall(EMAIL_REGEX, text)
|
| 57 |
+
phones = re.findall(PHONE_REGEX, text)
|
| 58 |
|
| 59 |
data = []
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
+
for email in set(emails):
|
| 62 |
+
idx = text.find(email)
|
| 63 |
+
context = text[max(0, idx-150):idx+50]
|
| 64 |
|
| 65 |
+
name = extract_name(context)
|
| 66 |
+
designation = extract_designation(context)
|
|
|
|
|
|
|
| 67 |
|
| 68 |
+
phone_match = re.search(PHONE_REGEX, context)
|
| 69 |
+
phone = phone_match.group() if phone_match else "Not Available"
|
| 70 |
|
| 71 |
+
data.append([name, email, phone, designation])
|
| 72 |
|
| 73 |
+
# -------- FIND MORE LINKS --------
|
| 74 |
+
links = []
|
| 75 |
+
for a in soup.find_all("a", href=True):
|
| 76 |
+
link = urljoin(url, a["href"])
|
| 77 |
|
| 78 |
+
if "faculty" in link.lower() or "profile" in link.lower():
|
| 79 |
+
links.append(link)
|
| 80 |
|
| 81 |
+
return data, list(set(links))
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
|
| 84 |
+
# -------- MAIN CRAWLER --------
|
| 85 |
+
def crawl_site(start_url, max_pages=5):
|
| 86 |
+
visited = set()
|
| 87 |
+
to_visit = [start_url]
|
| 88 |
|
| 89 |
+
all_data = []
|
| 90 |
|
| 91 |
+
while to_visit and len(visited) < max_pages:
|
| 92 |
+
url = to_visit.pop(0)
|
|
|
|
|
|
|
| 93 |
|
| 94 |
+
if url in visited:
|
| 95 |
+
continue
|
| 96 |
|
| 97 |
+
visited.add(url)
|
| 98 |
+
|
| 99 |
+
data, links = extract_from_page(url)
|
| 100 |
+
all_data.extend(data)
|
| 101 |
+
|
| 102 |
+
for link in links:
|
| 103 |
+
if link not in visited:
|
| 104 |
+
to_visit.append(link)
|
| 105 |
+
|
| 106 |
+
return pd.DataFrame(all_data, columns=["Name", "Email", "Phone", "Designation"])
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
# -------- CLEAN DATA --------
|
| 110 |
+
def clean_data(df):
|
| 111 |
+
cleaned = []
|
| 112 |
+
|
| 113 |
+
for email, group in df.groupby("Email"):
|
| 114 |
+
names = [n for n in group["Name"] if n]
|
| 115 |
+
phones = group["Phone"].tolist()
|
| 116 |
+
designations = group["Designation"].tolist()
|
| 117 |
+
|
| 118 |
+
best_name = max(names, key=len) if names else "Not Available"
|
| 119 |
+
best_phone = next((p for p in phones if p != "Not Available"), "Not Available")
|
| 120 |
+
best_designation = next((d for d in designations if d != "Not Available"), "Not Available")
|
| 121 |
|
| 122 |
+
cleaned.append([best_name, email, best_phone, best_designation])
|
|
|
|
|
|
|
|
|
|
| 123 |
|
| 124 |
+
return pd.DataFrame(cleaned, columns=["Name", "Email", "Phone", "Designation"])
|
|
|
|
|
|
|
|
|
|
| 125 |
|
|
|
|
| 126 |
|
| 127 |
# -------- UI --------
|
| 128 |
url = st.text_input("π Enter Faculty Page URL")
|
|
|
|
| 131 |
if not url:
|
| 132 |
st.warning("Enter a URL")
|
| 133 |
else:
|
| 134 |
+
with st.spinner("Crawling and extracting..."):
|
| 135 |
+
df = crawl_site(url)
|
| 136 |
+
df = clean_data(df)
|
| 137 |
|
| 138 |
if not df.empty:
|
| 139 |
st.success("β
Data Extracted Successfully")
|
|
|
|
| 148 |
"text/csv"
|
| 149 |
)
|
| 150 |
else:
|
| 151 |
+
st.error("No data found")
|