File size: 4,067 Bytes
6ca8e48
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
"""
Paste/upload ingester.

Accepts raw HTML, PDF (via Docling), or plain Markdown from the user
and converts it to clean Markdown suitable for Gemini structuring.
"""

import logging
import re

logger = logging.getLogger(__name__)


async def ingest_paste(content: str, content_type: str = "auto") -> str:
    """
    Process pasted or uploaded content into clean Markdown.

    content_type: "html" | "markdown" | "pdf_base64" | "auto"
    """
    if content_type == "auto":
        content_type = _detect_content_type(content)

    logger.info("Processing paste content as %s (%d chars)", content_type, len(content))

    if content_type == "html":
        return _html_to_markdown(content)
    elif content_type == "pdf_base64":
        return await _pdf_to_markdown(content)
    else:
        # Already Markdown or plain text — pass through with minimal cleanup
        return _clean_markdown(content)


def _detect_content_type(content: str) -> str:
    """Heuristic detection of content format."""
    stripped = content.strip()
    if stripped.startswith("<!DOCTYPE") or stripped.startswith("<html") or stripped.count("<div") > 3:
        return "html"
    # Base64-encoded PDF starts with JVBERi0
    if stripped.startswith("JVBERi0"):
        return "pdf_base64"
    return "markdown"


def _html_to_markdown(html: str) -> str:
    """
    Convert HTML to Markdown.
    Uses html2text if available, otherwise falls back to regex stripping.
    """
    try:
        import html2text  # type: ignore[import]
        h = html2text.HTML2Text()
        h.ignore_links = False
        h.ignore_images = True
        h.body_width = 0  # Don't wrap lines
        return h.handle(html)
    except ImportError:
        logger.debug("html2text not installed, using regex fallback")
        return _regex_html_strip(html)


def _regex_html_strip(html: str) -> str:
    """Minimal HTML → text via regex (fallback when html2text unavailable)."""
    html = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
    # Convert headings
    for i in range(6, 0, -1):
        html = re.sub(rf"<h{i}[^>]*>(.*?)</h{i}>", rf"{'#' * i} \1\n", html, flags=re.IGNORECASE | re.DOTALL)
    # Code blocks
    html = re.sub(r"<pre[^>]*>(.*?)</pre>", r"```\n\1\n```", html, flags=re.IGNORECASE | re.DOTALL)
    html = re.sub(r"<code[^>]*>(.*?)</code>", r"`\1`", html, flags=re.IGNORECASE | re.DOTALL)
    # Lists
    html = re.sub(r"<li[^>]*>(.*?)</li>", r"- \1", html, flags=re.IGNORECASE | re.DOTALL)
    # Paragraphs and divs
    html = re.sub(r"<(p|div|br)[^>]*>", "\n", html, flags=re.IGNORECASE)
    # Strip remaining tags
    html = re.sub(r"<[^>]+>", "", html)
    # Decode entities
    html = html.replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&").replace("&nbsp;", " ")
    return re.sub(r"\n{3,}", "\n\n", html).strip()


async def _pdf_to_markdown(pdf_base64: str) -> str:
    """
    Convert base64-encoded PDF to Markdown using Docling by IBM.
    Falls back to a message if Docling is not installed.
    """
    try:
        import base64
        import io

        from docling.document_converter import DocumentConverter  # type: ignore[import]

        pdf_bytes = base64.b64decode(pdf_base64)
        converter = DocumentConverter()
        # Docling expects a file path or URL; write to temp file
        import tempfile
        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
            f.write(pdf_bytes)
            tmp_path = f.name

        result = converter.convert(tmp_path)
        return result.document.export_to_markdown()
    except ImportError:
        logger.warning("Docling not installed, cannot convert PDF")
        return "[PDF conversion requires Docling: pip install docling]"
    except Exception as exc:
        logger.error("PDF conversion failed: %s", exc)
        return f"[PDF conversion failed: {exc}]"


def _clean_markdown(text: str) -> str:
    """Remove excessive blank lines and leading/trailing whitespace."""
    return re.sub(r"\n{4,}", "\n\n\n", text).strip()