contiai / apps /api /app /rag /html_text.py
aliroya's picture
ContiAI platform: API and queue worker in one container
57fa15f verified
Raw History Blame Contribute Delete
4.12 kB
"""Reading a source that is a web page rather than a PDF.
Ingest could only consume PDFs. Of forty-five sources selected for one book,
forty-one were discarded as `no_pdf_or_blocked` - not because they were blocked
but because they were HTML: journal landing pages, institutional articles, an
IBM report. Four documents reached the index, twenty-eight chunks reached the
writer, and one subsection in thirty-six carried a citation.
The search step already decided these were worth keeping: it ranks a web page
below a PDF instead of discarding it, precisely so a specialist article is not
lost to its file format. Ingest is where that decision was being reversed.
What comes back is the article, not the page: navigation, scripts, styles,
forms and comment threads are removed first, because embedding a site's menu
puts its menu in the index. A page that yields too little after that is
reported as too thin rather than indexed as prose.
"""
from __future__ import annotations
import re
#: Below this the page had no article on it - a landing page, a search result,
#: a paywall notice. A journal's abstract page cleared 1,200 characters on its
#: template alone (`##plugins.themes.bootstrap3.article.sidebar##`), which is
#: not text a book can be written from.
MIN_ARTICLE_CHARS = 2000
#: Everything that is furniture on any site.
_STRIP_TAGS = (
"script", "style", "nav", "header", "footer", "aside", "form",
"noscript", "iframe", "svg", "button", "figure",
)
#: Where an article usually lives, in the order worth trying.
_ARTICLE_HINTS = (
"article",
"main",
'[role="main"]',
".article-body",
".entry-content",
".post-content",
"#content",
)
_BLANK_RUN = re.compile(r"\n{3,}")
_ANY_BREAK = re.compile(r"\n+")
_SPACE_RUN = re.compile(r"[ \t ]{2,}")
def _clean(text: str) -> str:
text = _SPACE_RUN.sub(" ", str(text or ""))
lines = [line.strip() for line in text.splitlines()]
return _BLANK_RUN.sub("\n\n", "\n".join(lines)).strip()
def article_text(html: str) -> str:
"""The readable article in a page, or "" when there is none."""
from bs4 import BeautifulSoup
soup = BeautifulSoup(str(html or ""), "html.parser")
for tag in soup(list(_STRIP_TAGS)):
tag.decompose()
# The narrowest container that still holds the article. Falling straight
# to <body> picks up sidebars and related-article lists, which read as
# prose to every check downstream and are not this document's text.
for selector in _ARTICLE_HINTS:
found = soup.select_one(selector)
if found is not None:
text = _clean(found.get_text("\n"))
if len(text) >= MIN_ARTICLE_CHARS:
return text
body = soup.body or soup
return _clean(body.get_text("\n"))
def pages_from_html(html: str, chars_per_page: int = 2600) -> list[str]:
"""The article split into page-sized pieces.
A web page has no pages, and every citation downstream carries a page
range. Splitting on a fixed size gives a stable, honest number: "page 3"
of this document means the third block of its text, and the same block
every time the document is read.
Split on paragraph boundaries so a citation never points at half a
sentence.
"""
text = article_text(html)
if len(text) < MIN_ARTICLE_CHARS:
return []
# On any newline, not only a blank one. `get_text("\n")` puts a single
# newline between block elements, so splitting on blank lines saw a whole
# article as one paragraph and returned it as a single page - which makes
# every citation from that source point at "page 1" of forty pages.
paragraphs = [p.strip() for p in _ANY_BREAK.split(text) if p.strip()]
pages: list[str] = []
current: list[str] = []
size = 0
for paragraph in paragraphs:
if size and size + len(paragraph) > chars_per_page:
pages.append("\n\n".join(current))
current, size = [], 0
current.append(paragraph)
size += len(paragraph) + 2
if current:
pages.append("\n\n".join(current))
return pages