Spaces:
Sleeping
Sleeping
| import requests | |
| from bs4 import BeautifulSoup | |
| import re | |
| from urllib.parse import urljoin | |
| import gradio as gr | |
| # --------------------------------- | |
| # Validate proper email | |
| # --------------------------------- | |
| def is_valid_email(email): | |
| return re.match(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", email) | |
| # --------------------------------- | |
| # Clean text (remove garbage) | |
| # --------------------------------- | |
| def is_clean_text(text): | |
| if not text: | |
| return False | |
| if len(text) > 100: | |
| return False | |
| if any(char in text for char in ["�", "\x00"]): | |
| return False | |
| if any(word in text.lower() for word in [ | |
| "obj", "endobj", "stream", "pdf" | |
| ]): | |
| return False | |
| return True | |
| # --------------------------------- | |
| # Validate real name | |
| # --------------------------------- | |
| def is_valid_name(text): | |
| text_lower = text.lower() | |
| if any(word in text_lower for word in [ | |
| "iit", "university", "college", | |
| "department", "school", | |
| "faculty", "staff", "research", | |
| "principal", "administrator" | |
| ]): | |
| return False | |
| if "@" in text: | |
| return False | |
| words = text.split() | |
| return 2 <= len(words) <= 5 | |
| # --------------------------------- | |
| # Email → Name fallback | |
| # --------------------------------- | |
| def name_from_email(email): | |
| username = email.split("@")[0] | |
| parts = re.split(r"[._]", username) | |
| if len(parts) == 1: | |
| parts = re.findall('[A-Za-z][^A-Z]*', username) | |
| return " ".join([p.capitalize() for p in parts if p]) | |
| # --------------------------------- | |
| # Find closest name to email | |
| # --------------------------------- | |
| def find_name_near_email(text_node): | |
| parent = text_node.parent | |
| # Check previous siblings (closest first) | |
| for sibling in parent.previous_siblings: | |
| if hasattr(sibling, "get_text"): | |
| text = sibling.get_text(strip=True) | |
| if is_valid_name(text) and is_clean_text(text): | |
| return text | |
| # Check inside same parent | |
| text = parent.get_text("\n", strip=True) | |
| lines = text.split("\n") | |
| for line in lines: | |
| line = line.strip() | |
| if is_valid_name(line) and is_clean_text(line): | |
| return line | |
| return None | |
| # --------------------------------- | |
| # Main scraper | |
| # --------------------------------- | |
| def get_professor_details(college_url): | |
| try: | |
| headers = {"User-Agent": "Mozilla/5.0"} | |
| response = requests.get(college_url, headers=headers, timeout=10) | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # Find faculty-related links | |
| keywords = ["faculty", "people", "staff", "directory"] | |
| faculty_links = [] | |
| for link in soup.find_all("a", href=True): | |
| href = link["href"].lower() | |
| if any(word in href for word in keywords): | |
| faculty_links.append(urljoin(college_url, link["href"])) | |
| pages_to_scan = faculty_links if faculty_links else [college_url] | |
| results = [] | |
| seen = set() | |
| for page in pages_to_scan: | |
| try: | |
| res = requests.get(page, headers=headers, timeout=10) | |
| page_soup = BeautifulSoup(res.text, "html.parser") | |
| # Find all text nodes containing emails | |
| for text_node in page_soup.find_all(string=re.compile(r"@")): | |
| emails = re.findall(r"[\w\.-]+@[\w\.-]+", text_node) | |
| for email in emails: | |
| if not is_valid_email(email): | |
| continue | |
| if email in seen: | |
| continue | |
| seen.add(email) | |
| # Try to find real name near email | |
| name = find_name_near_email(text_node) | |
| # Fallback | |
| if not name: | |
| name = name_from_email(email) | |
| results.append({ | |
| "name": name, | |
| "email": email | |
| }) | |
| except: | |
| continue | |
| if not results: | |
| return "No data found." | |
| # Format output | |
| output = "" | |
| for r in results: | |
| output += f"Name: {r['name']}\nEmail: {r['email']}\n\n" | |
| return output | |
| except Exception as e: | |
| return f"Error: {str(e)}" | |
| # --------------------------------- | |
| # Gradio UI | |
| # --------------------------------- | |
| iface = gr.Interface( | |
| fn=get_professor_details, | |
| inputs=gr.Textbox(label="Enter College Website URL"), | |
| outputs=gr.Textbox(label="Professor Details"), | |
| title="🎓 College Professor Scraper (Final Version)", | |
| description="Extracts professor names and emails using hybrid intelligent scraping" | |
| ) | |
| iface.launch() |