Buckets:

download
raw
2.87 kB
"""
Lightweight text extractors for .rst and .tex content.
Prefer optional libraries (docutils, pylatexenc) when available. Fall back to
best-effort heuristics otherwise to avoid heavy optional dependencies in CI.
"""
from typing import Optional
import re
def extract_text_from_rst(rst: str) -> str:
"""Convert reStructuredText to plain text.
Uses docutils if available; otherwise applies simple heuristics to strip
common RST markup (directives, roles, inline markup).
"""
try:
from docutils.core import publish_string
# docutils returns bytes when writer_name='pseudoxml' or 'plaintext'
out = publish_string(rst, writer_name="plaintext")
if isinstance(out, bytes):
try:
return out.decode("utf-8").strip()
except Exception:
return out.decode(errors="ignore").strip()
return str(out).strip()
except Exception:
# Fallback heuristic
text = rst
# Remove reST directives (lines starting with ".. ") and their content blocks
text = re.sub(r"(?m)^\.\. .*$(?:\n^(?:\s+.*$))*", "", text)
# Remove role markup :role:`text`
text = re.sub(r":\w+:`([^`]+)`", r"\1", text)
# Replace inline literal backticks ``like this`` or `like this`
text = re.sub(r"``([^`]+)``", r"\1", text)
text = re.sub(r"`([^`]+)`", r"\1", text)
# Remove simple interpreted text (``..``) and emphasis/strong markers
text = text.replace("**", "").replace("*", "")
# Remove heading underlines/overlines (common in rst)
text = re.sub(r"(?m)^[-=~`\^\"']{2,}\s*$", "", text)
# Collapse multiple blank lines
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def extract_text_from_tex(tex: str) -> str:
"""Convert LaTeX content into plain text.
Uses pylatexenc if available; otherwise applies heuristic stripping of
macros and environments.
"""
try:
from pylatexenc.latex2text import LatexNodes2Text
return LatexNodes2Text().latex_to_text(tex).strip()
except Exception:
text = tex
# Remove comments
text = re.sub(r"%.*$", "", text, flags=re.MULTILINE)
# Keep inner text of common environments: replace \begin{env}...\end{env} with the inner content
text = re.sub(r"\\begin\{[^}]+\}(.*?)\\end\{[^}]+\}", r"\1", text, flags=re.DOTALL)
# Replace commands like \command[opt]{arg} or \command{arg} with their argument text
text = re.sub(r"\\[a-zA-Z]+\*?(?:\[[^\]]*\])?\{([^}]*)\}", r"\1", text)
# Remove remaining simple commands like \command
text = re.sub(r"\\[a-zA-Z]+\*?(?:\[[^\]]*\])?", "", text)
# Collapse multiple whitespace
text = re.sub(r"\s+", " ", text)
return text.strip()
__all__ = ["extract_text_from_rst", "extract_text_from_tex"]

Xet Storage Details

Size:
2.87 kB
·
Xet hash:
90c5c69a027582e4ffa7295e0048a7ea0986592f3ef6dd9bfaccd42f7a9706ce

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.