from bs4 import BeautifulSoup, Comment import logger def cleanup(soup): # Remove all HTML comments for comment in soup.find_all(string=lambda text: isinstance(text, Comment)): comment.extract() # Remove unnecessary tags from the
for meta in soup.select('head meta'): if not meta.has_attr('charset') and not meta.has_attr('http-equiv'): meta.decompose() # Remove all inline CSS styles for tag in soup.find_all(True): if tag.has_attr('style'): del tag['style'] # Remove any empty , , , elements for tag in soup.select('b:empty, i:empty, u:empty, sup:empty'): tag.decompose() # Remove unnecessary attributes like valign and align from tags for tag in soup.find_all(True): for attr in ['valign', 'align']: if tag.has_attr(attr): del tag[attr] # Remove all elements that just contain page numbers for span in soup.select('span.page'): if span.get_text(strip=True).isdigit(): span.decompose() return soup def parse(filepath): soup = cleanup(BeautifulSoup(open(filepath).read(), 'html.parser')) logger.info(f"Parsed {filepath}") return soup