Download apps/api/app/rag/html_text.py from aliroya/contiai: direct link, hf CLI and curl.
- Browser
- Download file 4.12 kB
-
https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/app/rag/html_text.py
- Command line
-
hf download hf://spaces/aliroya/contiai/apps/api/app/rag/html_text.py
-
curl -L -o html_text.py https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/app/rag/html_text.py
4.12 kB
| """Reading a source that is a web page rather than a PDF. | |
| Ingest could only consume PDFs. Of forty-five sources selected for one book, | |
| forty-one were discarded as `no_pdf_or_blocked` - not because they were blocked | |
| but because they were HTML: journal landing pages, institutional articles, an | |
| IBM report. Four documents reached the index, twenty-eight chunks reached the | |
| writer, and one subsection in thirty-six carried a citation. | |
| The search step already decided these were worth keeping: it ranks a web page | |
| below a PDF instead of discarding it, precisely so a specialist article is not | |
| lost to its file format. Ingest is where that decision was being reversed. | |
| What comes back is the article, not the page: navigation, scripts, styles, | |
| forms and comment threads are removed first, because embedding a site's menu | |
| puts its menu in the index. A page that yields too little after that is | |
| reported as too thin rather than indexed as prose. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| #: Below this the page had no article on it - a landing page, a search result, | |
| #: a paywall notice. A journal's abstract page cleared 1,200 characters on its | |
| #: template alone (`##plugins.themes.bootstrap3.article.sidebar##`), which is | |
| #: not text a book can be written from. | |
| MIN_ARTICLE_CHARS = 2000 | |
| #: Everything that is furniture on any site. | |
| _STRIP_TAGS = ( | |
| "script", "style", "nav", "header", "footer", "aside", "form", | |
| "noscript", "iframe", "svg", "button", "figure", | |
| ) | |
| #: Where an article usually lives, in the order worth trying. | |
| _ARTICLE_HINTS = ( | |
| "article", | |
| "main", | |
| '[role="main"]', | |
| ".article-body", | |
| ".entry-content", | |
| ".post-content", | |
| "#content", | |
| ) | |
| _BLANK_RUN = re.compile(r"\n{3,}") | |
| _ANY_BREAK = re.compile(r"\n+") | |
| _SPACE_RUN = re.compile(r"[ \t ]{2,}") | |
| def _clean(text: str) -> str: | |
| text = _SPACE_RUN.sub(" ", str(text or "")) | |
| lines = [line.strip() for line in text.splitlines()] | |
| return _BLANK_RUN.sub("\n\n", "\n".join(lines)).strip() | |
| def article_text(html: str) -> str: | |
| """The readable article in a page, or "" when there is none.""" | |
| from bs4 import BeautifulSoup | |
| soup = BeautifulSoup(str(html or ""), "html.parser") | |
| for tag in soup(list(_STRIP_TAGS)): | |
| tag.decompose() | |
| # The narrowest container that still holds the article. Falling straight | |
| # to <body> picks up sidebars and related-article lists, which read as | |
| # prose to every check downstream and are not this document's text. | |
| for selector in _ARTICLE_HINTS: | |
| found = soup.select_one(selector) | |
| if found is not None: | |
| text = _clean(found.get_text("\n")) | |
| if len(text) >= MIN_ARTICLE_CHARS: | |
| return text | |
| body = soup.body or soup | |
| return _clean(body.get_text("\n")) | |
| def pages_from_html(html: str, chars_per_page: int = 2600) -> list[str]: | |
| """The article split into page-sized pieces. | |
| A web page has no pages, and every citation downstream carries a page | |
| range. Splitting on a fixed size gives a stable, honest number: "page 3" | |
| of this document means the third block of its text, and the same block | |
| every time the document is read. | |
| Split on paragraph boundaries so a citation never points at half a | |
| sentence. | |
| """ | |
| text = article_text(html) | |
| if len(text) < MIN_ARTICLE_CHARS: | |
| return [] | |
| # On any newline, not only a blank one. `get_text("\n")` puts a single | |
| # newline between block elements, so splitting on blank lines saw a whole | |
| # article as one paragraph and returned it as a single page - which makes | |
| # every citation from that source point at "page 1" of forty pages. | |
| paragraphs = [p.strip() for p in _ANY_BREAK.split(text) if p.strip()] | |
| pages: list[str] = [] | |
| current: list[str] = [] | |
| size = 0 | |
| for paragraph in paragraphs: | |
| if size and size + len(paragraph) > chars_per_page: | |
| pages.append("\n\n".join(current)) | |
| current, size = [], 0 | |
| current.append(paragraph) | |
| size += len(paragraph) + 2 | |
| if current: | |
| pages.append("\n\n".join(current)) | |
| return pages | |