"""Reading a source that is a web page rather than a PDF. Ingest could only consume PDFs. Of forty-five sources selected for one book, forty-one were discarded as `no_pdf_or_blocked` - not because they were blocked but because they were HTML: journal landing pages, institutional articles, an IBM report. Four documents reached the index, twenty-eight chunks reached the writer, and one subsection in thirty-six carried a citation. The search step already decided these were worth keeping: it ranks a web page below a PDF instead of discarding it, precisely so a specialist article is not lost to its file format. Ingest is where that decision was being reversed. What comes back is the article, not the page: navigation, scripts, styles, forms and comment threads are removed first, because embedding a site's menu puts its menu in the index. A page that yields too little after that is reported as too thin rather than indexed as prose. """ from __future__ import annotations import re #: Below this the page had no article on it - a landing page, a search result, #: a paywall notice. A journal's abstract page cleared 1,200 characters on its #: template alone (`##plugins.themes.bootstrap3.article.sidebar##`), which is #: not text a book can be written from. MIN_ARTICLE_CHARS = 2000 #: Everything that is furniture on any site. _STRIP_TAGS = ( "script", "style", "nav", "header", "footer", "aside", "form", "noscript", "iframe", "svg", "button", "figure", ) #: Where an article usually lives, in the order worth trying. _ARTICLE_HINTS = ( "article", "main", '[role="main"]', ".article-body", ".entry-content", ".post-content", "#content", ) _BLANK_RUN = re.compile(r"\n{3,}") _ANY_BREAK = re.compile(r"\n+") _SPACE_RUN = re.compile(r"[ \t ]{2,}") def _clean(text: str) -> str: text = _SPACE_RUN.sub(" ", str(text or "")) lines = [line.strip() for line in text.splitlines()] return _BLANK_RUN.sub("\n\n", "\n".join(lines)).strip() def article_text(html: str) -> str: """The readable article in a page, or "" when there is none.""" from bs4 import BeautifulSoup soup = BeautifulSoup(str(html or ""), "html.parser") for tag in soup(list(_STRIP_TAGS)): tag.decompose() # The narrowest container that still holds the article. Falling straight # to picks up sidebars and related-article lists, which read as # prose to every check downstream and are not this document's text. for selector in _ARTICLE_HINTS: found = soup.select_one(selector) if found is not None: text = _clean(found.get_text("\n")) if len(text) >= MIN_ARTICLE_CHARS: return text body = soup.body or soup return _clean(body.get_text("\n")) def pages_from_html(html: str, chars_per_page: int = 2600) -> list[str]: """The article split into page-sized pieces. A web page has no pages, and every citation downstream carries a page range. Splitting on a fixed size gives a stable, honest number: "page 3" of this document means the third block of its text, and the same block every time the document is read. Split on paragraph boundaries so a citation never points at half a sentence. """ text = article_text(html) if len(text) < MIN_ARTICLE_CHARS: return [] # On any newline, not only a blank one. `get_text("\n")` puts a single # newline between block elements, so splitting on blank lines saw a whole # article as one paragraph and returned it as a single page - which makes # every citation from that source point at "page 1" of forty pages. paragraphs = [p.strip() for p in _ANY_BREAK.split(text) if p.strip()] pages: list[str] = [] current: list[str] = [] size = 0 for paragraph in paragraphs: if size and size + len(paragraph) > chars_per_page: pages.append("\n\n".join(current)) current, size = [], 0 current.append(paragraph) size += len(paragraph) + 2 if current: pages.append("\n\n".join(current)) return pages