Spaces:
Running
Running
File size: 4,417 Bytes
df2f2e8 db7ec76 df2f2e8 db7ec76 df2f2e8 db7ec76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | import os
import time
import requests
from bs4 import BeautifulSoup
from typing import List, Dict, Optional
from urllib.parse import urljoin
import src.scraper.profile_scraper as profile_scraper
from summarize import summarize
import json
class FacultyScraper:
def __init__(self, base_url: str, delay: float = 0.1, debug: bool = False):
self.base_url = base_url
self.delay = delay
self.debug = debug
self.response = requests.get(self.base_url)
self.soup = BeautifulSoup(self.response.text, 'html.parser')
def save_to_jsonl(self, output_file: str = 'faculty_data.jsonl') -> None:
"""
Save faculty information to a JSONL file.
Args:
output_file (str): Name of the output JSONL file
"""
if not self.faculty_list:
print("No faculty data to save. Please scrape data first.")
return
# remove the file if it exists
if os.path.exists(output_file):
os.remove(output_file)
with open(output_file, 'w', encoding='utf-8') as f:
for faculty_member in self.faculty_list:
f.write(json.dumps(faculty_member, ensure_ascii=False) + '\n')
print(f"Saved data for {len(self.faculty_list)} faculty members to {output_file}")
def scrape_faculty_list(self) -> List[Dict]:
"""
Scrape faculty information from the provided HTML content.
Args:
html_content (str): HTML content containing faculty information
Returns:
List[Dict]: List of dictionaries containing faculty information
"""
faculty_list = []
# Find all faculty divs with class 'cell fac-sort'
faculty_divs = self._get_faculty_divs()
if self.debug:
faculty_divs = faculty_divs[:5]
print(f"Debug: Only scraping first 5 faculty members")
for i, faculty_div in enumerate(faculty_divs):
faculty_info = {}
# Add delay between each professor
if i > 0:
time.sleep(self.delay)
# Extract image URL
image_url = self._extract_image_url(faculty_div)
if image_url:
faculty_info['image_url'] = image_url
# Extract name and profile information
name = self._extract_name(faculty_div)
if name:
faculty_info['name'] = name
profile_url = self._extract_profile_url(faculty_div)
if profile_url:
profile_summary = self._get_faculty_profile(profile_url)
faculty_info['profile'] = profile_summary
faculty_info['profile_url'] = profile_url
faculty_list.append(faculty_info)
print(f"Processed {i+1}/{len(faculty_divs)} faculty members")
self.faculty_list = faculty_list
def _get_faculty_profile(self, profile_url: str) -> str:
"""
Get faculty profile summary using the summarize function.
Args:
profile_url (str): URL of the faculty profile
Returns:
str: Profile summary
"""
try:
return profile_scraper.FacultyProfileScraper(profile_url).text
except Exception as e:
print(f"Error summarizing profile {profile_url}: {e}")
return f"Error: Could not summarize profile"
def _get_faculty_divs(self) -> List:
'''
Get the faculty divs from the soup
'''
pass
def _extract_image_url(self, faculty_div) -> Optional[str]:
'''
Extract the image url from the faculty div
'''
pass
def _extract_name(self, faculty_div) -> Dict:
'''
Extract the name from the faculty div
'''
pass
def _extract_profile_url(self, faculty_div) -> Dict:
pass
def get_absolute_url(self, base_url: str, relative_url: str) -> str:
"""
Convert a relative URL to an absolute URL.
Args:
base_url (str): The base URL of the website
relative_url (str): The relative URL to convert
Returns:
str: The absolute URL
"""
return urljoin(base_url, relative_url)
|