DU / parse.py
Dmitry Kirsanov
redo logging
d465175
Raw History Blame Contribute Delete
1.27 kB
from bs4 import BeautifulSoup, Comment
import logger
def cleanup(soup):
# Remove all HTML comments
for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
comment.extract()
# Remove unnecessary <meta> tags from the <head>
for meta in soup.select('head meta'):
if not meta.has_attr('charset') and not meta.has_attr('http-equiv'):
meta.decompose()
# Remove all inline CSS styles
for tag in soup.find_all(True):
if tag.has_attr('style'):
del tag['style']
# Remove any empty <b>, <i>, <u>, <sup> elements
for tag in soup.select('b:empty, i:empty, u:empty, sup:empty'):
tag.decompose()
# Remove unnecessary attributes like valign and align from tags
for tag in soup.find_all(True):
for attr in ['valign', 'align']:
if tag.has_attr(attr):
del tag[attr]
# Remove all <span class="page"> elements that just contain page numbers
for span in soup.select('span.page'):
if span.get_text(strip=True).isdigit():
span.decompose()
return soup
def parse(filepath):
soup = cleanup(BeautifulSoup(open(filepath).read(), 'html.parser'))
logger.info(f"Parsed {filepath}")
return soup