Spaces:
Runtime error
Runtime error
Download parse.py from PrecisionContent/DU: direct link, hf CLI and curl.
- Browser
- Download file 1.27 kB
-
https://huggingface.co/spaces/PrecisionContent/DU/resolve/main/parse.py
- Command line
-
hf download hf://spaces/PrecisionContent/DU/parse.py
-
curl -L -o parse.py https://huggingface.co/spaces/PrecisionContent/DU/resolve/main/parse.py
1.27 kB
| from bs4 import BeautifulSoup, Comment | |
| import logger | |
| def cleanup(soup): | |
| # Remove all HTML comments | |
| for comment in soup.find_all(string=lambda text: isinstance(text, Comment)): | |
| comment.extract() | |
| # Remove unnecessary <meta> tags from the <head> | |
| for meta in soup.select('head meta'): | |
| if not meta.has_attr('charset') and not meta.has_attr('http-equiv'): | |
| meta.decompose() | |
| # Remove all inline CSS styles | |
| for tag in soup.find_all(True): | |
| if tag.has_attr('style'): | |
| del tag['style'] | |
| # Remove any empty <b>, <i>, <u>, <sup> elements | |
| for tag in soup.select('b:empty, i:empty, u:empty, sup:empty'): | |
| tag.decompose() | |
| # Remove unnecessary attributes like valign and align from tags | |
| for tag in soup.find_all(True): | |
| for attr in ['valign', 'align']: | |
| if tag.has_attr(attr): | |
| del tag[attr] | |
| # Remove all <span class="page"> elements that just contain page numbers | |
| for span in soup.select('span.page'): | |
| if span.get_text(strip=True).isdigit(): | |
| span.decompose() | |
| return soup | |
| def parse(filepath): | |
| soup = cleanup(BeautifulSoup(open(filepath).read(), 'html.parser')) | |
| logger.info(f"Parsed {filepath}") | |
| return soup | |