Upload crawler/parser.py with huggingface_hub

eeb5dd1 verified 6 months ago

3.63 kB

import re
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
from typing import Dict, Any, List, Set

class PageParser:
    def __init__(self, domain: str, scheme: str = "https"):
        self.domain = domain.lower()
        self.scheme = scheme

    def clean_text(self, text: str) -> str:
        """Cleans formatting, extra spaces, and redundant newlines from text."""
        if not text:
            return ""
        # Remove script and style elements content during parsing
        text = re.sub(r'\s+', ' ', text)
        return text.strip()

    def parse(self, html_content: str, current_url: str) -> Dict[str, Any]:
        """Extracts structured text data and meta information from the HTML."""
        # Use lxml parser for fast and memory efficient parsing
        soup = BeautifulSoup(html_content, "lxml")
        
        # Strip script, style, and iframe components before extracting text
        for element in soup(["script", "style", "iframe", "noscript"]):
            element.decompose()

        # Page metadata
        title = soup.title.string.strip() if soup.title else ""
        
        meta_desc = ""
        meta_desc_tag = soup.find("meta", attrs={"name": "description"}) or soup.find("meta", attrs={"property": "og:description"})
        if meta_desc_tag:
            meta_desc = meta_desc_tag.get("content", "").strip()

        meta_keys = ""
        meta_keys_tag = soup.find("meta", attrs={"name": "keywords"})
        if meta_keys_tag:
            meta_keys = meta_keys_tag.get("content", "").strip()

        # Header structures
        h1s = [self.clean_text(h.get_text()) for h in soup.find_all("h1") if h.get_text()]
        h1_headers = " | ".join(h1s)

        # Body main text extraction
        # Focus on paragraphs, main, article sections if possible, fallback to clean body text
        main_content = soup.find("main") or soup.find("article") or soup.find("body")
        if main_content:
            text_content = self.clean_text(main_content.get_text(separator=" "))
        else:
            text_content = self.clean_text(soup.get_text(separator=" "))

        return {
            "title": title,
            "meta_description": meta_desc,
            "meta_keywords": meta_keys,
            "h1_headers": h1_headers,
            "text_content": text_content,
        }

    def extract_links(self, html_content: str, current_url: str) -> Set[str]:
        """Discovers and normalizes all internal/same-domain links from page anchor tags."""
        soup = BeautifulSoup(html_content, "lxml")
        links = set()
        
        for anchor in soup.find_all("a", href=True):
            href = anchor["href"].strip()
            
            # Skip empty, mailto, javascript, and anchor links
            if not href or href.startswith(("#", "mailto:", "javascript:", "tel:")):
                continue
                
            # Resolve relative URLs
            absolute_url = urljoin(current_url, href)
            
            # Parse resolved URL
            parsed = urlparse(absolute_url)
            
            # Verify same domain
            if parsed.netloc.lower() == self.domain:
                # Normalize url by stripping fragment (#) and query params sorting if needed
                normalized_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
                # Append trailing slash consistency or query parameters if it contains pathing parameters
                if parsed.query:
                    normalized_url += f"?{parsed.query}"
                links.add(normalized_url)
                
        return links

	import re
	from urllib.parse import urljoin, urlparse
	from bs4 import BeautifulSoup
	from typing import Dict, Any, List, Set

	class PageParser:
	def __init__(self, domain: str, scheme: str = "https"):
	self.domain = domain.lower()
	self.scheme = scheme

	def clean_text(self, text: str) -> str:
	"""Cleans formatting, extra spaces, and redundant newlines from text."""
	if not text:
	return ""
	# Remove script and style elements content during parsing
	text = re.sub(r'\s+', ' ', text)
	return text.strip()

	def parse(self, html_content: str, current_url: str) -> Dict[str, Any]:
	"""Extracts structured text data and meta information from the HTML."""
	# Use lxml parser for fast and memory efficient parsing
	soup = BeautifulSoup(html_content, "lxml")

	# Strip script, style, and iframe components before extracting text
	for element in soup(["script", "style", "iframe", "noscript"]):
	element.decompose()

	# Page metadata
	title = soup.title.string.strip() if soup.title else ""

	meta_desc = ""
	meta_desc_tag = soup.find("meta", attrs={"name": "description"}) or soup.find("meta", attrs={"property": "og:description"})
	if meta_desc_tag:
	meta_desc = meta_desc_tag.get("content", "").strip()

	meta_keys = ""
	meta_keys_tag = soup.find("meta", attrs={"name": "keywords"})
	if meta_keys_tag:
	meta_keys = meta_keys_tag.get("content", "").strip()

	# Header structures
	h1s = [self.clean_text(h.get_text()) for h in soup.find_all("h1") if h.get_text()]
	h1_headers = " \| ".join(h1s)

	# Body main text extraction
	# Focus on paragraphs, main, article sections if possible, fallback to clean body text
	main_content = soup.find("main") or soup.find("article") or soup.find("body")
	if main_content:
	text_content = self.clean_text(main_content.get_text(separator=" "))
	else:
	text_content = self.clean_text(soup.get_text(separator=" "))

	return {
	"title": title,
	"meta_description": meta_desc,
	"meta_keywords": meta_keys,
	"h1_headers": h1_headers,
	"text_content": text_content,
	}

	def extract_links(self, html_content: str, current_url: str) -> Set[str]:
	"""Discovers and normalizes all internal/same-domain links from page anchor tags."""
	soup = BeautifulSoup(html_content, "lxml")
	links = set()

	for anchor in soup.find_all("a", href=True):
	href = anchor["href"].strip()

	# Skip empty, mailto, javascript, and anchor links
	if not href or href.startswith(("#", "mailto:", "javascript:", "tel:")):
	continue

	# Resolve relative URLs
	absolute_url = urljoin(current_url, href)

	# Parse resolved URL
	parsed = urlparse(absolute_url)

	# Verify same domain
	if parsed.netloc.lower() == self.domain:
	# Normalize url by stripping fragment (#) and query params sorting if needed
	normalized_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
	# Append trailing slash consistency or query parameters if it contains pathing parameters
	if parsed.query:
	normalized_url += f"?{parsed.query}"
	links.add(normalized_url)

	return links