File size: 1,038 Bytes
1c0a11d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
import re
from io import BytesIO

import pdfplumber


SUPPORTED_MIME = ["application/pdf", "text/markdown", "text/plain"]


def parse_file(content: bytes, mime: str) -> str:
    if mime == "application/pdf":
        text = []
        with pdfplumber.open(BytesIO(content)) as pdf:
            for page in pdf.pages:
                page_text = page.extract_text()
                if page_text:
                    text.append(page_text.strip())
        return "\n\n".join(text)

    if mime in ("text/markdown", "text/plain"):
        raw = content.decode("utf-8", errors="ignore")
        if mime == "text/markdown":
            raw = re.sub(r"#{1,6}\s*", "", raw)
            raw = re.sub(r"\*\*(.+?)\*\*", r"\1", raw)
            raw = re.sub(r"\*(.+?)\*", r"\1", raw)
            raw = re.sub(r"`{1,3}.*?`{1,3}", "", raw, flags=re.DOTALL)
            raw = re.sub(r"!\[.*?\]\(.*?\)", "", raw)
            raw = re.sub(r"\[(.+?)\]\(.*?\)", r"\1", raw)
        return raw.strip()

    raise ValueError(f"Unsupported mime type: {mime}")