"""Document Ingestion & Chunking Module. Supports: - PDF (layout-aware, tables extracted to Markdown, OCR fallback for scanned pages) - DOCX (paragraphs, headings hierarchy, structured tables) - TXT (markdown-style headings, paragraph structure) - Semantic, structure-aware chunking (300-500 tokens, 10-15% overlap) - Metadata preservation (page number, section title, table flags, document ID) """ import os import re import uuid from dataclasses import dataclass, field, asdict from typing import List, Dict, Any, Optional, Tuple try: import tiktoken _TIKTOKEN_AVAILABLE = True _ENCODER = tiktoken.get_encoding("cl100k_base") except Exception: _TIKTOKEN_AVAILABLE = False _ENCODER = None # PDF Processing try: import pdfplumber _PDFPLUMBER_AVAILABLE = True except ImportError: _PDFPLUMBER_AVAILABLE = False try: import pypdf _PYPDF_AVAILABLE = True except ImportError: _PYPDF_AVAILABLE = False # DOCX Processing try: import docx _DOCX_AVAILABLE = True except ImportError: _DOCX_AVAILABLE = False # OCR Processing try: import pytesseract from pdf2image import convert_from_path _OCR_AVAILABLE = True except ImportError: _OCR_AVAILABLE = False @dataclass class DocumentChunk: chunk_id: str doc_id: str file_name: str text: str page_number: int section_title: str chunk_index: int token_count: int metadata: Dict[str, Any] = field(default_factory=dict) def to_dict(self) -> Dict[str, Any]: return asdict(self) @classmethod def from_dict(cls, data: Dict[str, Any]) -> "DocumentChunk": return cls(**data) @dataclass class ParsedDocument: doc_id: str file_name: str file_type: str total_pages: int chunks: List[DocumentChunk] metadata: Dict[str, Any] = field(default_factory=dict) def count_tokens(text: str) -> int: """Accurately count tokens using tiktoken cl100k_base or word fallback.""" if _TIKTOKEN_AVAILABLE and _ENCODER is not None: return len(_ENCODER.encode(text, disallowed_special=())) # Fallback approximation: 1 token ~ 0.75 words / 4 chars return max(1, int(len(text.split()) * 1.3)) def table_to_markdown(table_rows: List[List[Optional[str]]]) -> str: """Converts a 2D list of cells into a clean GitHub-flavored Markdown table.""" if not table_rows or not any(table_rows): return "" cleaned_rows: List[List[str]] = [] for row in table_rows: if not row: continue cleaned_rows.append([str(c or "").strip().replace("\n", " ") for c in row]) if not cleaned_rows: return "" num_cols = max(len(r) for r in cleaned_rows) # Pad rows to have uniform column count for r in cleaned_rows: while len(r) < num_cols: r.append("") header = cleaned_rows[0] separator = ["---"] * num_cols body = cleaned_rows[1:] if len(cleaned_rows) > 1 else [] lines = [ "| " + " | ".join(header) + " |", "| " + " | ".join(separator) + " |", ] for r in body: lines.append("| " + " | ".join(r) + " |") return "\n".join(lines) class DocumentIngestionPipeline: def __init__( self, chunk_size: int = 400, chunk_overlap: int = 50, min_chunk_size: int = 40, ocr_fallback: bool = True, ocr_min_chars: int = 40, ): self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap self.min_chunk_size = min_chunk_size self.ocr_fallback = ocr_fallback self.ocr_min_chars = ocr_min_chars def ingest_file( self, file_path: str, doc_id: Optional[str] = None, original_filename: Optional[str] = None ) -> ParsedDocument: """Main entry point to parse and chunk a document.""" if not os.path.exists(file_path): raise FileNotFoundError(f"File not found: {file_path}") file_name = original_filename or os.path.basename(file_path) ext = os.path.splitext(file_name)[1].lower() if not doc_id: doc_id = str(uuid.uuid4())[:8] # Ensure working path has proper extension for third-party parsers actual_path = file_path temp_target = None if not file_path.lower().endswith(ext) and ext: import shutil temp_target = f"{file_path}_{uuid.uuid4().hex[:6]}{ext}" shutil.copyfile(file_path, temp_target) actual_path = temp_target try: if ext == ".pdf": raw_elements, total_pages = self._parse_pdf(actual_path) file_type = "pdf" elif ext in [".docx", ".doc"]: raw_elements, total_pages = self._parse_docx(actual_path) file_type = "docx" elif ext in [".txt", ".md"]: raw_elements, total_pages = self._parse_txt(actual_path) file_type = "txt" else: raise ValueError(f"Unsupported file type: {ext}. Supported: .pdf, .docx, .txt, .md") # Chunk parsed elements with structure awareness chunks = self._chunk_elements(raw_elements, doc_id, file_name) finally: if temp_target and os.path.exists(temp_target): try: os.remove(temp_target) except Exception: pass return ParsedDocument( doc_id=doc_id, file_name=file_name, file_type=file_type, total_pages=total_pages, chunks=chunks, metadata={"file_path": file_path, "total_chunks": len(chunks)} ) def _parse_pdf(self, file_path: str) -> Tuple[List[Dict[str, Any]], int]: """Extract text and tables from PDF, with OCR fallback for scanned pages.""" elements = [] total_pages = 0 # Try pdfplumber first for high fidelity layout & table extraction if _PDFPLUMBER_AVAILABLE: with pdfplumber.open(file_path) as pdf: total_pages = len(pdf.pages) current_section = "Introduction" # Pre-load pypdf reader for clean word boundaries and whitespace pypdf_reader = None if _PYPDF_AVAILABLE: try: pypdf_reader = pypdf.PdfReader(file_path) except Exception: pypdf_reader = None for page_idx, page in enumerate(pdf.pages): page_num = page_idx + 1 page_elements = [] # 1. Extract tables extracted_tables = [] try: tables = page.extract_tables() for table in tables: md_table = table_to_markdown(table) if md_table: extracted_tables.append(md_table) except Exception: pass # 2. Extract text: prefer pypdf for clean token separation and proper spacing raw_text = "" if pypdf_reader is not None and page_idx < len(pypdf_reader.pages): try: raw_text = pypdf_reader.pages[page_idx].extract_text() or "" except Exception: raw_text = "" # Fallback to pdfplumber text if pypdf was empty if not raw_text.strip(): raw_text = page.extract_text(layout=False) or "" # 3. OCR Fallback for scanned/empty pages is_ocr = False if len(raw_text.strip()) < self.ocr_min_chars and self.ocr_fallback and _OCR_AVAILABLE: try: # Convert single page to image and OCR images = convert_from_path( file_path, first_page=page_num, last_page=page_num, dpi=200 ) if images: ocr_text = pytesseract.image_to_string(images[0]) if len(ocr_text.strip()) > len(raw_text.strip()): raw_text = ocr_text is_ocr = True except Exception: # If OCR tools (e.g. tesseract/poppler) fail, keep original text pass # Process paragraphs and detect headings lines = raw_text.splitlines() para_buf = [] for line in lines: clean_line = line.strip() if not clean_line: if para_buf: para_text = " ".join(para_buf) page_elements.append({ "text": para_text, "page": page_num, "section": current_section, "is_table": False, "is_ocr": is_ocr }) para_buf = [] continue # Heading heuristics: Short, title-case/all-caps, or numbered is_heading = False if ( (len(clean_line) < 70 and not clean_line.endswith((".", ","))) and ( clean_line.isupper() or re.match(r"^(Chapter|\d+(\.\d+)*)\b", clean_line, re.IGNORECASE) or clean_line.startswith("#") ) ): is_heading = True if is_heading: if para_buf: para_text = " ".join(para_buf) page_elements.append({ "text": para_text, "page": page_num, "section": current_section, "is_table": False, "is_ocr": is_ocr }) para_buf = [] current_section = clean_line.lstrip("#").strip() else: para_buf.append(clean_line) if para_buf: page_elements.append({ "text": " ".join(para_buf), "page": page_num, "section": current_section, "is_table": False, "is_ocr": is_ocr }) # Add extracted tables for tbl in extracted_tables: page_elements.append({ "text": tbl, "page": page_num, "section": current_section, "is_table": True, "is_ocr": is_ocr }) elements.extend(page_elements) elif _PYPDF_AVAILABLE: reader = pypdf.PdfReader(file_path) total_pages = len(reader.pages) current_section = "General" for idx, page in enumerate(reader.pages): page_num = idx + 1 text = page.extract_text() or "" elements.append({ "text": text, "page": page_num, "section": current_section, "is_table": False, "is_ocr": False }) else: raise ImportError("Neither pdfplumber nor pypdf is installed for PDF parsing.") return elements, total_pages def _parse_docx(self, file_path: str) -> Tuple[List[Dict[str, Any]], int]: """Extract paragraphs, headings, and tables from DOCX with approximate page indexing.""" if not _DOCX_AVAILABLE: raise ImportError("python-docx is not installed.") doc = docx.Document(file_path) elements = [] current_section = "Overview" current_page = 1 words_on_page = 0 WORDS_PER_PAGE = 350 # Average words per standard page for element in doc.element.body: # Check if paragraph if element.tag.endswith("p"): # Find matching paragraph object p_text = "" p_style = "" for p in doc.paragraphs: if p._p == element: p_text = p.text.strip() p_style = p.style.name if p.style else "" break if not p_text: continue if "Heading" in p_style or "Title" in p_style: current_section = p_text continue w_count = len(p_text.split()) words_on_page += w_count if words_on_page > WORDS_PER_PAGE: current_page += 1 words_on_page = w_count elements.append({ "text": p_text, "page": current_page, "section": current_section, "is_table": False, "is_ocr": False }) # Check if table elif element.tag.endswith("tbl"): for t in doc.tables: if t._tbl == element: table_data = [] for row in t.rows: table_data.append([cell.text.strip() for cell in row.cells]) md_table = table_to_markdown(table_data) if md_table: elements.append({ "text": md_table, "page": current_page, "section": current_section, "is_table": True, "is_ocr": False }) break total_pages = max(current_page, 1) return elements, total_pages def _parse_txt(self, file_path: str) -> Tuple[List[Dict[str, Any]], int]: """Extract text and markdown-style headings from plain text or markdown.""" with open(file_path, "r", encoding="utf-8", errors="replace") as f: content = f.read() lines = content.splitlines() elements = [] current_section = "General" current_page = 1 para_buf = [] words_on_page = 0 WORDS_PER_PAGE = 350 for line in lines: clean = line.strip() if not clean: if para_buf: text_block = " ".join(para_buf) w_count = len(text_block.split()) words_on_page += w_count if words_on_page > WORDS_PER_PAGE: current_page += 1 words_on_page = w_count elements.append({ "text": text_block, "page": current_page, "section": current_section, "is_table": False, "is_ocr": False }) para_buf = [] continue # Heading detection if clean.startswith("#"): if para_buf: elements.append({ "text": " ".join(para_buf), "page": current_page, "section": current_section, "is_table": False, "is_ocr": False }) para_buf = [] current_section = clean.lstrip("#").strip() elif clean.isupper() and len(clean) < 60: if para_buf: elements.append({ "text": " ".join(para_buf), "page": current_page, "section": current_section, "is_table": False, "is_ocr": False }) para_buf = [] current_section = clean else: para_buf.append(clean) if para_buf: elements.append({ "text": " ".join(para_buf), "page": current_page, "section": current_section, "is_table": False, "is_ocr": False }) total_pages = max(current_page, 1) return elements, total_pages def _chunk_elements( self, raw_elements: List[Dict[str, Any]], doc_id: str, file_name: str ) -> List[DocumentChunk]: """ Structure-aware semantic chunking. - Keeps page number and section title as metadata. - Respects table boundaries (does not slice small tables). - Target chunk size: 300 - 500 tokens. - Overlap: 10 - 15% (e.g. 50 tokens). """ chunks: List[DocumentChunk] = [] chunk_idx = 0 # Group elements by (section, page) when logical, or accumulate up to chunk_size current_tokens: List[str] = [] current_page = 1 current_section = "General" is_table = False is_ocr = False for elem in raw_elements: elem_text = elem["text"].strip() if not elem_text: continue elem_page = elem["page"] elem_section = elem["section"] elem_is_table = elem.get("is_table", False) elem_is_ocr = elem.get("is_ocr", False) # If element is a table and fits in chunk_size, keep as dedicated chunk elem_tokens_count = count_tokens(elem_text) if elem_is_table and elem_tokens_count <= self.chunk_size: # Flush existing buffer first if current_tokens: c_text = " ".join(current_tokens) if c_text.strip(): chunk_id = f"{doc_id}_{chunk_idx:04d}" chunks.append(DocumentChunk( chunk_id=chunk_id, doc_id=doc_id, file_name=file_name, text=c_text, page_number=current_page, section_title=current_section, chunk_index=chunk_idx, token_count=count_tokens(c_text), metadata={"is_table": is_table, "is_ocr": is_ocr} )) chunk_idx += 1 current_tokens = [] # Emit table chunk chunk_id = f"{doc_id}_{chunk_idx:04d}" chunks.append(DocumentChunk( chunk_id=chunk_id, doc_id=doc_id, file_name=file_name, text=elem_text, page_number=elem_page, section_title=elem_section, chunk_index=chunk_idx, token_count=elem_tokens_count, metadata={"is_table": True, "is_ocr": elem_is_ocr} )) chunk_idx += 1 current_page = elem_page current_section = elem_section continue # If section changed and we have existing tokens, flush current chunk if elem_section != current_section and len(current_tokens) > 0: c_text = " ".join(current_tokens) if c_text.strip(): chunk_id = f"{doc_id}_{chunk_idx:04d}" chunks.append(DocumentChunk( chunk_id=chunk_id, doc_id=doc_id, file_name=file_name, text=c_text, page_number=current_page, section_title=current_section, chunk_index=chunk_idx, token_count=count_tokens(c_text), metadata={"is_table": is_table, "is_ocr": is_ocr} )) chunk_idx += 1 current_tokens = [] # Sentence/paragraph level splitting to build chunks sentences = re.split(r"(?<=[.!?])\s+", elem_text) for s in sentences: s = s.strip() if not s: continue # Check candidate size test_text = " ".join(current_tokens + [s]) test_token_count = count_tokens(test_text) if test_token_count <= self.chunk_size: current_tokens.append(s) current_page = elem_page current_section = elem_section is_ocr = elem_is_ocr or is_ocr else: # Flush current chunk if current_tokens: c_text = " ".join(current_tokens) if count_tokens(c_text) >= self.min_chunk_size: chunk_id = f"{doc_id}_{chunk_idx:04d}" chunks.append(DocumentChunk( chunk_id=chunk_id, doc_id=doc_id, file_name=file_name, text=c_text, page_number=current_page, section_title=current_section, chunk_index=chunk_idx, token_count=count_tokens(c_text), metadata={"is_table": is_table, "is_ocr": is_ocr} )) chunk_idx += 1 # Compute overlap tokens from end of previous chunk overlap_tokens: List[str] = [] overlap_count = 0 for prev_s in reversed(current_tokens): s_tokens = count_tokens(prev_s) if overlap_count + s_tokens <= self.chunk_overlap: overlap_tokens.insert(0, prev_s) overlap_count += s_tokens else: break current_tokens = overlap_tokens + [s] else: current_tokens = [s] current_page = elem_page current_section = elem_section is_ocr = elem_is_ocr # Flush final remaining tokens if current_tokens: c_text = " ".join(current_tokens) if c_text.strip(): chunk_id = f"{doc_id}_{chunk_idx:04d}" chunks.append(DocumentChunk( chunk_id=chunk_id, doc_id=doc_id, file_name=file_name, text=c_text, page_number=current_page, section_title=current_section, chunk_index=chunk_idx, token_count=count_tokens(c_text), metadata={"is_table": is_table, "is_ocr": is_ocr} )) return chunks