Blame for src/streamlit_app.py

import streamlit as st
import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
from urllib.parse import urljoin, urlparse
import time

# --- CONFIG & CONSTANTS ---
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": USER_AGENT}
EMAIL_REGEX = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
# Improved Phone Regex to avoid matching years/dates
PHONE_REGEX = r'(?:\+?\d{1,3}[-.\s]?)?\(?\d{3,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,4}'
TITLES = ["Dr", "Prof", "Mr", "Ms", "Mrs", "Shri", "Smt", "Doctor"]
DESIGNATIONS = [
    "Assistant Professor", "Associate Professor", "Professor", 
    "HOD", "Head of Department", "Lecturer", "Dean", 
    "Director", "Faculty", "Instructor", "Registrar"
]
# Keywords that invalidate a string as a Name
INVALID_NAME_TOKENS = [
    "phone", "email", "contact", "dept", "department", "office", "fax", 
    "mobile", "university", "college", "address", "profile", "research", 
    "publication", "student", "grievance", "cell", "ext", "tel", "call"
]

# --- UTILITY FUNCTIONS ---

def clean_text(text):
    if not text: return ""
    # Remove non-breaking spaces and extra whitespace
    text = text.replace('\xa0', ' ')
    return re.sub(r'\s+', ' ', text).strip()

def is_valid_name(name_str):
    if not name_str: return False
    clean = clean_text(name_str)
    
    # 1. Length constraint
    if not (3 <= len(clean) <= 60): return False
    
    # 2. Reject if contains digits
    if any(char.isdigit() for char in clean): return False
    
    # 3. Reject if contains invalid keywords (case-insensitive)
    lower_clean = clean.lower()
    if any(token in lower_clean for token in INVALID_NAME_TOKENS): return False
    
    # 4. Must contain at least one alphabet
    if not re.search(r'[a-zA-Z]', clean): return False
    
    # 5. Must have at least 2 words OR a known title
    words = clean.split()
    has_title = any(t.lower() in [w.lower().strip('.') for w in words] for t in TITLES)
    if len(words) >= 2 or has_title:
        return True
    
    return False

def score_name(name, base_confidence):
    if not name: return 0
    score = float(base_confidence)
    words = name.split()
    # Bonus for academic titles
    if any(t.lower() in [w.lower().strip('.') for w in words] for t in TITLES):
        score += 1.5
    # Bonus for standard name lengths (2-3 words)
    if 2 <= len(words) <= 3:
        score += 1.0
    # Penalty for very long strings
    if len(name) > 40:
        score -= 0.5
    return score

def extract_designation(text):
    text_lower = text.lower()
    # Sort designations by length descending to match "Assistant Professor" before "Professor"
    for desig in sorted(DESIGNATIONS, key=len, reverse=True):
        if desig.lower() in text_lower:
            return desig
    return ""

# --- SCRAPER ENGINE ---

class FacultyScraper:
    def __init__(self):
        self.visited_urls = set()
        self.candidates = [] # List of dicts: {email, name, phone, designation, score}

    def fetch_soup(self, url):
        try:
            response = requests.get(url, headers=HEADERS, timeout=15, verify=False)
            response.raise_for_status()
            return BeautifulSoup(response.text, 'html.parser')
        except Exception as e:
            return None

    def extract_from_soup(self, soup):
        if not soup: return

        # Strategy A: Block-based (High Confidence)
        # Look for containers that likely hold one person's info
        for block_tag in ['div', 'tr', 'li', 'section', 'article']:
            for block in soup.find_all(block_tag):
                # Only process blocks that contain an email
                block_text = block.get_text(" ", strip=True)
                emails = re.findall(EMAIL_REGEX, block_text)
                if not emails: continue
                
                email = emails[0].lower()
                # Find name: look for headings or bold text within this specific block
                name = ""
                for header in block.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'strong', 'b']):
                    possible = clean_text(header.get_text())
                    if is_valid_name(possible):
                        name = possible
                        break
                
                phone_match = re.search(PHONE_REGEX, block_text)
                phone = phone_match.group(0) if phone_match else ""
                desig = extract_designation(block_text)
                
                if name:
                    self.candidates.append({
                        "Email": email, "Name": name, "Phone": phone, 
                        "Designation": desig, "Score": score_name(name, 3)
                    })

        # Strategy B: Context-based (Medium Confidence)
        full_text = soup.get_text(" ", strip=True)
        for match in re.finditer(EMAIL_REGEX, full_text):
            email = match.group(0).lower()
            # Extract window around email
            start = max(0, match.start() - 150)
            end = min(len(full_text), match.end() + 150)
            context = full_text[start:end]
            
            # Logic: Name usually appears BEFORE the email in text
            pre_context = full_text[max(0, match.start()-100) : match.start()]
            lines = [l.strip() for l in pre_context.split('  ') if l.strip()]
            
            best_context_name = ""
            for line in reversed(lines):
                if is_valid_name(line):
                    best_context_name = line
                    break
            
            if best_context_name:
                self.candidates.append({
                    "Email": email, "Name": best_context_name, 
                    "Phone": re.search(PHONE_REGEX, context).group(0) if re.search(PHONE_REGEX, context) else "",
                    "Designation": extract_designation(context),
                    "Score": score_name(best_context_name, 2)
                })

        # Strategy C: Fallback (Low Confidence)
        all_emails = re.findall(EMAIL_REGEX, full_text)
        for email in all_emails:
            email = email.lower()
            username = email.split('@')[0]
            fallback_name = username.replace('.', ' ').replace('_', ' ').title()
            if len(fallback_name) > 3:
                self.candidates.append({
                    "Email": email, "Name": fallback_name, "Phone": "", 
                    "Designation": "", "Score": 1.0
                })

    def run(self, start_url):
        # 1. Crawl Logic
        self.visited_urls.add(start_url)
        soup = self.fetch_soup(start_url)
        if not soup: return pd.DataFrame(columns=["Name", "Email", "Phone", "Designation"])
        
        self.extract_from_soup(soup)
        
        # Follow links (Max 5 pages total)
        links = soup.find_all('a', href=True)
        crawled_count = 1
        for link in links:
            if crawled_count >= 5: break
            href = urljoin(start_url, link['href'])
            parsed = urlparse(href)
            # Stay on same domain, look for keywords
            if parsed.netloc == urlparse(start_url).netloc:
                if any(k in href.lower() for k in ["faculty", "staff", "profile", "people", "directory"]):
                    if href not in self.visited_urls:
                        self.visited_urls.add(href)
                        sub_soup = self.fetch_soup(href)
                        if sub_soup:
                            self.extract_from_soup(sub_soup)
                            crawled_count += 1

        # 2. Process Candidates
        if not self.candidates:
            return pd.DataFrame(columns=["Name", "Email", "Phone", "Designation"])

        df = pd.DataFrame(self.candidates)
        
        # 3. Aggressive Deduplication and Selection
        # Group by Email and pick the candidate with the highest score
        df = df.sort_values(by="Score", ascending=False)
        # Clean specific columns before final drop
        df['Name'] = df['Name'].apply(clean_text)
        df['Phone'] = df['Phone'].apply(clean_text)
        df['Designation'] = df['Designation'].apply(clean_text)
        
        # Remove records where Email is clearly invalid (optional but safer)
        df = df[df['Email'].str.contains(r'[^@]+@[^@]+\.[^@]+')]
        
        # Final unique selection per Email
        df_final = df.groupby('Email').first().reset_index()
        
        # Final column ordering
        return df_final[["Name", "Email", "Phone", "Designation"]]

# --- STREAMLIT UI ---

def main():
    st.set_page_config(page_title="Faculty Scraper Pro", layout="wide")
    st.title("🏛️ Production-Grade Faculty Extractor")
    st.markdown("Extracts clean, deduplicated faculty data using multi-strategy block analysis.")

    url = st.text_input("Enter Faculty Directory URL:", placeholder="https://www.university.edu/science/faculty")
    
    if st.button("Start Extraction", type="primary"):
        if not url:
            st.error("Please enter a URL first.")
            return
            
        with st.spinner("Analyzing pages and resolving duplicates..."):
            scraper = FacultyScraper()
            results_df = scraper.run(url)
            
            if not results_df.empty:
                st.success(f"Extracted {len(results_df)} unique faculty members.")
                
                # Cleanup display: Ensure no weird characters or messed up rows
                results_df = results_df.fillna("").astype(str)
                
                st.dataframe(results_df, use_container_width=True)
                
                csv = results_df.to_csv(index=False).encode('utf-8')
                st.download_button(
                    label="Download Data as CSV",
                    data=csv,
                    file_name="faculty_data.csv",
                    mime="text/csv",
                )
            else:
                st.warning("No faculty data found. Ensure the URL is a public directory page.")

    st.divider()
    st.caption("Built with Python, BeautifulSoup, and pandas. Designed for production-grade robustness.")

if __name__ == "__main__":
    # Disable warnings for self-signed certificates common in edu sites
    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
    main()