Spaces:
Sleeping
Sleeping
Download mrag/ingest.py from Angshul/Multimodal-RAG: direct link, hf CLI and curl.
- Browser
- Download file 8.58 kB
-
https://huggingface.co/spaces/Angshul/Multimodal-RAG/resolve/main/mrag/ingest.py
- Command line
-
hf download hf://spaces/Angshul/Multimodal-RAG/mrag/ingest.py
-
curl -L -o ingest.py https://huggingface.co/spaces/Angshul/Multimodal-RAG/resolve/main/mrag/ingest.py
8.58 kB
| """Document loaders: PDF (text, embedded images, scanned pages), DOCX (text, tables, images), | |
| images, and plain text/markdown. Everything is normalised into TextItem / ImageItem records.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import io | |
| import logging | |
| import re | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from PIL import Image | |
| log = logging.getLogger(__name__) | |
| IMAGE_EXT = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif", ".tif", ".tiff"} | |
| TEXT_EXT = {".txt", ".md", ".markdown", ".rst", ".csv"} | |
| SUPPORTED = IMAGE_EXT | TEXT_EXT | {".pdf", ".docx"} | |
| class TextItem: | |
| text: str | |
| source: str | |
| page: int | None = None | |
| kind: str = "text" # "text" | "table" | "ocr" | |
| image_id: str | None = None # set when the text came from an image (OCR) | |
| class ImageItem: | |
| image: Image.Image | |
| source: str | |
| page: int | None = None | |
| sha1: str = "" | |
| meta: dict = field(default_factory=dict) | |
| class Extracted: | |
| texts: list[TextItem] = field(default_factory=list) | |
| images: list[ImageItem] = field(default_factory=list) | |
| def sha1_bytes(b: bytes) -> str: | |
| return hashlib.sha1(b).hexdigest() | |
| def _clean(s: str) -> str: | |
| s = s.replace("", "") # soft hyphen | |
| s = re.sub(r"-\n(?=[a-z])", "", s) # de-hyphenate line breaks | |
| s = re.sub(r"[ \t]+", " ", s) | |
| s = re.sub(r"\n{3,}", "\n\n", s) | |
| return s.strip() | |
| def chunk_text(text: str, chunk_words: int = 180, overlap: int = 40) -> list[str]: | |
| """Paragraph-aware word-window chunking.""" | |
| if chunk_words <= 0 or not 0 <= overlap < chunk_words: | |
| raise ValueError("require chunk_words > 0 and 0 <= overlap < chunk_words") | |
| paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()] | |
| chunks: list[str] = [] | |
| buf: list[str] = [] | |
| step = max(1, chunk_words - overlap) | |
| for p in paras: | |
| words = p.split() | |
| if len(words) > chunk_words: # long paragraph: flush, then slide a window | |
| if buf: | |
| chunks.append(" ".join(buf)) | |
| buf = [] | |
| for i in range(0, len(words), step): | |
| chunks.append(" ".join(words[i:i + chunk_words])) | |
| if i + chunk_words >= len(words): | |
| break | |
| continue | |
| if len(buf) + len(words) > chunk_words and buf: | |
| chunks.append(" ".join(buf)) | |
| # Preserve only overlap that fits beside the next whole paragraph. | |
| keep = min(overlap, chunk_words - len(words)) | |
| buf = buf[-keep:] if keep else [] | |
| buf.extend(words) | |
| if buf: | |
| chunks.append(" ".join(buf)) | |
| return chunks | |
| def _img_from_bytes(b: bytes) -> Image.Image | None: | |
| try: | |
| im = Image.open(io.BytesIO(b)) | |
| im.load() | |
| if getattr(im, "n_frames", 1) > 1: | |
| im.seek(0) | |
| return im.convert("RGB") | |
| except Exception as e: # unsupported codec etc. | |
| log.warning("could not decode image: %s", e) | |
| return None | |
| def load_pdf(path: Path, min_px: int, scanned_chars: int, max_pages: int = 0) -> Extracted: | |
| import pymupdf as fitz | |
| out = Extracted() | |
| doc = fitz.open(path) | |
| seen: set[str] = set() | |
| for pno, page in enumerate(doc, start=1): | |
| if max_pages and pno > max_pages: | |
| log.warning("%s: only the first %d pages were indexed", path.name, max_pages) | |
| break | |
| text = _clean(page.get_text("text")) | |
| if len(text) >= scanned_chars: | |
| out.texts.append(TextItem(text, str(path), pno)) | |
| else: # scanned / image-only page: rasterise it, OCR will supply the text | |
| pix = page.get_pixmap(dpi=150) | |
| png = pix.tobytes("png") | |
| im = _img_from_bytes(png) | |
| if im is not None: | |
| out.images.append(ImageItem(im, str(path), pno, sha1_bytes(png), {"scanned_page": True})) | |
| continue | |
| # Vector graphics (plots, flowcharts, PDF-drawn tables) are not returned by | |
| # get_images(). Rasterise pages that contain drawing commands so retrieval | |
| # and the VLM can inspect diagrams even in born-digital scientific PDFs. | |
| # Do not OCR this extra rendering: its native text layer is already indexed. | |
| if page.get_drawings(): | |
| pix = page.get_pixmap(dpi=120, alpha=False) | |
| png = pix.tobytes("png") | |
| im = _img_from_bytes(png) | |
| if im is not None: | |
| out.images.append(ImageItem(im, str(path), pno, sha1_bytes(png), | |
| {"page_render": True})) | |
| for info in page.get_images(full=True): | |
| xref = info[0] | |
| try: | |
| raw = doc.extract_image(xref)["image"] | |
| im = _img_from_bytes(raw) | |
| except Exception: | |
| raw, im = b"", None | |
| if im is None: | |
| try: | |
| pix = fitz.Pixmap(doc, xref) | |
| if pix.n - pix.alpha >= 4: | |
| pix = fitz.Pixmap(fitz.csRGB, pix) | |
| raw = pix.tobytes("png") | |
| im = _img_from_bytes(raw) | |
| except Exception as e: | |
| log.warning("%s p%d: skipping image xref %d (%s)", path.name, pno, xref, e) | |
| continue | |
| if im is None or min(im.size) < min_px: | |
| continue | |
| h = sha1_bytes(raw) | |
| if f"{pno}:{h}" in seen: | |
| continue | |
| seen.add(f"{pno}:{h}") | |
| out.images.append(ImageItem(im, str(path), pno, h)) | |
| return out | |
| def load_docx(path: Path, min_px: int) -> Extracted: | |
| import docx | |
| from docx.oxml.ns import qn | |
| out = Extracted() | |
| d = docx.Document(str(path)) | |
| body: list[str] = [] | |
| # walk body in document order so tables stay where they were | |
| for el in d.element.body.iterchildren(): | |
| if el.tag == qn("w:p"): | |
| t = "".join(n.text or "" for n in el.iter(qn("w:t"))) | |
| if t.strip(): | |
| body.append(t.strip()) | |
| elif el.tag == qn("w:tbl"): | |
| rows = [] | |
| for tr in el.iter(qn("w:tr")): | |
| cells = [] | |
| for tc in tr.iter(qn("w:tc")): | |
| cells.append(" ".join("".join(n.text or "" for n in p.iter(qn("w:t"))) | |
| for p in tc.iter(qn("w:p"))).strip()) | |
| rows.append(" | ".join(cells)) | |
| if rows: | |
| out.texts.append(TextItem("Table:\n" + "\n".join(rows), str(path), None, "table")) | |
| if body: | |
| out.texts.insert(0, TextItem(_clean("\n\n".join(body)), str(path))) | |
| seen: set[str] = set() | |
| for rel in d.part.rels.values(): | |
| if "image" in rel.reltype and not rel.is_external: | |
| blob = rel.target_part.blob | |
| h = sha1_bytes(blob) | |
| if h in seen: | |
| continue | |
| seen.add(h) | |
| im = _img_from_bytes(blob) | |
| if im is not None and min(im.size) >= min_px: | |
| out.images.append(ImageItem(im, str(path), None, h)) | |
| return out | |
| def load_image(path: Path, min_px: int) -> Extracted: | |
| raw = path.read_bytes() | |
| im = _img_from_bytes(raw) | |
| out = Extracted() | |
| if im is not None and min(im.size) >= min_px: | |
| out.images.append(ImageItem(im, str(path), None, sha1_bytes(raw))) | |
| return out | |
| def load_text(path: Path) -> Extracted: | |
| return Extracted(texts=[TextItem(_clean(path.read_text(encoding="utf-8", errors="replace")), str(path))]) | |
| def load_any(path: Path, min_px: int = 64, scanned_chars: int = 30, max_pages: int = 0) -> Extracted: | |
| ext = path.suffix.lower() | |
| if ext == ".pdf": | |
| return load_pdf(path, min_px, scanned_chars, max_pages) | |
| if ext == ".docx": | |
| return load_docx(path, min_px) | |
| if ext in IMAGE_EXT: | |
| return load_image(path, min_px) | |
| if ext in TEXT_EXT: | |
| return load_text(path) | |
| raise ValueError(f"unsupported file type: {path}") | |
| def iter_files(paths: list[str | Path]): | |
| for p in map(Path, paths): | |
| if p.is_dir(): | |
| for f in sorted(p.rglob("*")): | |
| if f.is_file() and f.suffix.lower() in SUPPORTED and not any( | |
| part.startswith(".") for part in f.relative_to(p).parts): | |
| yield f | |
| elif p.is_file(): | |
| if p.suffix.lower() in SUPPORTED: | |
| yield p | |
| else: | |
| log.warning("skipping unsupported file %s", p) | |
| else: | |
| raise FileNotFoundError(p) | |