Multimodal-RAG / mrag /ingest.py
Angshul's picture
Upload 42 files
c42f4ef verified
Raw History Blame Contribute Delete
8.58 kB
"""Document loaders: PDF (text, embedded images, scanned pages), DOCX (text, tables, images),
images, and plain text/markdown. Everything is normalised into TextItem / ImageItem records."""
from __future__ import annotations
import hashlib
import io
import logging
import re
from dataclasses import dataclass, field
from pathlib import Path
from PIL import Image
log = logging.getLogger(__name__)
IMAGE_EXT = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif", ".tif", ".tiff"}
TEXT_EXT = {".txt", ".md", ".markdown", ".rst", ".csv"}
SUPPORTED = IMAGE_EXT | TEXT_EXT | {".pdf", ".docx"}
@dataclass
class TextItem:
text: str
source: str
page: int | None = None
kind: str = "text" # "text" | "table" | "ocr"
image_id: str | None = None # set when the text came from an image (OCR)
@dataclass
class ImageItem:
image: Image.Image
source: str
page: int | None = None
sha1: str = ""
meta: dict = field(default_factory=dict)
@dataclass
class Extracted:
texts: list[TextItem] = field(default_factory=list)
images: list[ImageItem] = field(default_factory=list)
def sha1_bytes(b: bytes) -> str:
return hashlib.sha1(b).hexdigest()
def _clean(s: str) -> str:
s = s.replace("­", "") # soft hyphen
s = re.sub(r"-\n(?=[a-z])", "", s) # de-hyphenate line breaks
s = re.sub(r"[ \t]+", " ", s)
s = re.sub(r"\n{3,}", "\n\n", s)
return s.strip()
def chunk_text(text: str, chunk_words: int = 180, overlap: int = 40) -> list[str]:
"""Paragraph-aware word-window chunking."""
if chunk_words <= 0 or not 0 <= overlap < chunk_words:
raise ValueError("require chunk_words > 0 and 0 <= overlap < chunk_words")
paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
chunks: list[str] = []
buf: list[str] = []
step = max(1, chunk_words - overlap)
for p in paras:
words = p.split()
if len(words) > chunk_words: # long paragraph: flush, then slide a window
if buf:
chunks.append(" ".join(buf))
buf = []
for i in range(0, len(words), step):
chunks.append(" ".join(words[i:i + chunk_words]))
if i + chunk_words >= len(words):
break
continue
if len(buf) + len(words) > chunk_words and buf:
chunks.append(" ".join(buf))
# Preserve only overlap that fits beside the next whole paragraph.
keep = min(overlap, chunk_words - len(words))
buf = buf[-keep:] if keep else []
buf.extend(words)
if buf:
chunks.append(" ".join(buf))
return chunks
def _img_from_bytes(b: bytes) -> Image.Image | None:
try:
im = Image.open(io.BytesIO(b))
im.load()
if getattr(im, "n_frames", 1) > 1:
im.seek(0)
return im.convert("RGB")
except Exception as e: # unsupported codec etc.
log.warning("could not decode image: %s", e)
return None
def load_pdf(path: Path, min_px: int, scanned_chars: int, max_pages: int = 0) -> Extracted:
import pymupdf as fitz
out = Extracted()
doc = fitz.open(path)
seen: set[str] = set()
for pno, page in enumerate(doc, start=1):
if max_pages and pno > max_pages:
log.warning("%s: only the first %d pages were indexed", path.name, max_pages)
break
text = _clean(page.get_text("text"))
if len(text) >= scanned_chars:
out.texts.append(TextItem(text, str(path), pno))
else: # scanned / image-only page: rasterise it, OCR will supply the text
pix = page.get_pixmap(dpi=150)
png = pix.tobytes("png")
im = _img_from_bytes(png)
if im is not None:
out.images.append(ImageItem(im, str(path), pno, sha1_bytes(png), {"scanned_page": True}))
continue
# Vector graphics (plots, flowcharts, PDF-drawn tables) are not returned by
# get_images(). Rasterise pages that contain drawing commands so retrieval
# and the VLM can inspect diagrams even in born-digital scientific PDFs.
# Do not OCR this extra rendering: its native text layer is already indexed.
if page.get_drawings():
pix = page.get_pixmap(dpi=120, alpha=False)
png = pix.tobytes("png")
im = _img_from_bytes(png)
if im is not None:
out.images.append(ImageItem(im, str(path), pno, sha1_bytes(png),
{"page_render": True}))
for info in page.get_images(full=True):
xref = info[0]
try:
raw = doc.extract_image(xref)["image"]
im = _img_from_bytes(raw)
except Exception:
raw, im = b"", None
if im is None:
try:
pix = fitz.Pixmap(doc, xref)
if pix.n - pix.alpha >= 4:
pix = fitz.Pixmap(fitz.csRGB, pix)
raw = pix.tobytes("png")
im = _img_from_bytes(raw)
except Exception as e:
log.warning("%s p%d: skipping image xref %d (%s)", path.name, pno, xref, e)
continue
if im is None or min(im.size) < min_px:
continue
h = sha1_bytes(raw)
if f"{pno}:{h}" in seen:
continue
seen.add(f"{pno}:{h}")
out.images.append(ImageItem(im, str(path), pno, h))
return out
def load_docx(path: Path, min_px: int) -> Extracted:
import docx
from docx.oxml.ns import qn
out = Extracted()
d = docx.Document(str(path))
body: list[str] = []
# walk body in document order so tables stay where they were
for el in d.element.body.iterchildren():
if el.tag == qn("w:p"):
t = "".join(n.text or "" for n in el.iter(qn("w:t")))
if t.strip():
body.append(t.strip())
elif el.tag == qn("w:tbl"):
rows = []
for tr in el.iter(qn("w:tr")):
cells = []
for tc in tr.iter(qn("w:tc")):
cells.append(" ".join("".join(n.text or "" for n in p.iter(qn("w:t")))
for p in tc.iter(qn("w:p"))).strip())
rows.append(" | ".join(cells))
if rows:
out.texts.append(TextItem("Table:\n" + "\n".join(rows), str(path), None, "table"))
if body:
out.texts.insert(0, TextItem(_clean("\n\n".join(body)), str(path)))
seen: set[str] = set()
for rel in d.part.rels.values():
if "image" in rel.reltype and not rel.is_external:
blob = rel.target_part.blob
h = sha1_bytes(blob)
if h in seen:
continue
seen.add(h)
im = _img_from_bytes(blob)
if im is not None and min(im.size) >= min_px:
out.images.append(ImageItem(im, str(path), None, h))
return out
def load_image(path: Path, min_px: int) -> Extracted:
raw = path.read_bytes()
im = _img_from_bytes(raw)
out = Extracted()
if im is not None and min(im.size) >= min_px:
out.images.append(ImageItem(im, str(path), None, sha1_bytes(raw)))
return out
def load_text(path: Path) -> Extracted:
return Extracted(texts=[TextItem(_clean(path.read_text(encoding="utf-8", errors="replace")), str(path))])
def load_any(path: Path, min_px: int = 64, scanned_chars: int = 30, max_pages: int = 0) -> Extracted:
ext = path.suffix.lower()
if ext == ".pdf":
return load_pdf(path, min_px, scanned_chars, max_pages)
if ext == ".docx":
return load_docx(path, min_px)
if ext in IMAGE_EXT:
return load_image(path, min_px)
if ext in TEXT_EXT:
return load_text(path)
raise ValueError(f"unsupported file type: {path}")
def iter_files(paths: list[str | Path]):
for p in map(Path, paths):
if p.is_dir():
for f in sorted(p.rglob("*")):
if f.is_file() and f.suffix.lower() in SUPPORTED and not any(
part.startswith(".") for part in f.relative_to(p).parts):
yield f
elif p.is_file():
if p.suffix.lower() in SUPPORTED:
yield p
else:
log.warning("skipping unsupported file %s", p)
else:
raise FileNotFoundError(p)