Spaces:
Runtime error
Runtime error
File size: 1,272 Bytes
4309ee9 4cf70e0 d465175 097e81a 4cf70e0 097e81a d465175 0b4b0ed 4cf70e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | from bs4 import BeautifulSoup, Comment
import logger
def cleanup(soup):
# Remove all HTML comments
for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
comment.extract()
# Remove unnecessary <meta> tags from the <head>
for meta in soup.select('head meta'):
if not meta.has_attr('charset') and not meta.has_attr('http-equiv'):
meta.decompose()
# Remove all inline CSS styles
for tag in soup.find_all(True):
if tag.has_attr('style'):
del tag['style']
# Remove any empty <b>, <i>, <u>, <sup> elements
for tag in soup.select('b:empty, i:empty, u:empty, sup:empty'):
tag.decompose()
# Remove unnecessary attributes like valign and align from tags
for tag in soup.find_all(True):
for attr in ['valign', 'align']:
if tag.has_attr(attr):
del tag[attr]
# Remove all <span class="page"> elements that just contain page numbers
for span in soup.select('span.page'):
if span.get_text(strip=True).isdigit():
span.decompose()
return soup
def parse(filepath):
soup = cleanup(BeautifulSoup(open(filepath).read(), 'html.parser'))
logger.info(f"Parsed {filepath}")
return soup
|