"""
Paste/upload ingester.
Accepts raw HTML, PDF (via Docling), or plain Markdown from the user
and converts it to clean Markdown suitable for Gemini structuring.
"""
import logging
import re
logger = logging.getLogger(__name__)
async def ingest_paste(content: str, content_type: str = "auto") -> str:
"""
Process pasted or uploaded content into clean Markdown.
content_type: "html" | "markdown" | "pdf_base64" | "auto"
"""
if content_type == "auto":
content_type = _detect_content_type(content)
logger.info("Processing paste content as %s (%d chars)", content_type, len(content))
if content_type == "html":
return _html_to_markdown(content)
elif content_type == "pdf_base64":
return await _pdf_to_markdown(content)
else:
# Already Markdown or plain text — pass through with minimal cleanup
return _clean_markdown(content)
def _detect_content_type(content: str) -> str:
"""Heuristic detection of content format."""
stripped = content.strip()
if stripped.startswith(" 3:
return "html"
# Base64-encoded PDF starts with JVBERi0
if stripped.startswith("JVBERi0"):
return "pdf_base64"
return "markdown"
def _html_to_markdown(html: str) -> str:
"""
Convert HTML to Markdown.
Uses html2text if available, otherwise falls back to regex stripping.
"""
try:
import html2text # type: ignore[import]
h = html2text.HTML2Text()
h.ignore_links = False
h.ignore_images = True
h.body_width = 0 # Don't wrap lines
return h.handle(html)
except ImportError:
logger.debug("html2text not installed, using regex fallback")
return _regex_html_strip(html)
def _regex_html_strip(html: str) -> str:
"""Minimal HTML → text via regex (fallback when html2text unavailable)."""
html = re.sub(r"<(script|style)[^>]*>.*?\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
# Convert headings
for i in range(6, 0, -1):
html = re.sub(rf"
]*>(.*?)", r"```\n\1\n```", html, flags=re.IGNORECASE | re.DOTALL) html = re.sub(r"
]*>(.*?)", r"`\1`", html, flags=re.IGNORECASE | re.DOTALL)
# Lists
html = re.sub(r"