Spaces:
Running on Zero
Running on Zero
Download ingest.py from Varun2007/document-qa-system: direct link, hf CLI and curl.
- Browser
- Download file 24.5 kB
-
https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/ingest.py
- Command line
-
hf download hf://spaces/Varun2007/document-qa-system/ingest.py
-
curl -L -o ingest.py https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/ingest.py
24.5 kB
| """Document Ingestion & Chunking Module. | |
| Supports: | |
| - PDF (layout-aware, tables extracted to Markdown, OCR fallback for scanned pages) | |
| - DOCX (paragraphs, headings hierarchy, structured tables) | |
| - TXT (markdown-style headings, paragraph structure) | |
| - Semantic, structure-aware chunking (300-500 tokens, 10-15% overlap) | |
| - Metadata preservation (page number, section title, table flags, document ID) | |
| """ | |
| import os | |
| import re | |
| import uuid | |
| from dataclasses import dataclass, field, asdict | |
| from typing import List, Dict, Any, Optional, Tuple | |
| try: | |
| import tiktoken | |
| _TIKTOKEN_AVAILABLE = True | |
| _ENCODER = tiktoken.get_encoding("cl100k_base") | |
| except Exception: | |
| _TIKTOKEN_AVAILABLE = False | |
| _ENCODER = None | |
| # PDF Processing | |
| try: | |
| import pdfplumber | |
| _PDFPLUMBER_AVAILABLE = True | |
| except ImportError: | |
| _PDFPLUMBER_AVAILABLE = False | |
| try: | |
| import pypdf | |
| _PYPDF_AVAILABLE = True | |
| except ImportError: | |
| _PYPDF_AVAILABLE = False | |
| # DOCX Processing | |
| try: | |
| import docx | |
| _DOCX_AVAILABLE = True | |
| except ImportError: | |
| _DOCX_AVAILABLE = False | |
| # OCR Processing | |
| try: | |
| import pytesseract | |
| from pdf2image import convert_from_path | |
| _OCR_AVAILABLE = True | |
| except ImportError: | |
| _OCR_AVAILABLE = False | |
| class DocumentChunk: | |
| chunk_id: str | |
| doc_id: str | |
| file_name: str | |
| text: str | |
| page_number: int | |
| section_title: str | |
| chunk_index: int | |
| token_count: int | |
| metadata: Dict[str, Any] = field(default_factory=dict) | |
| def to_dict(self) -> Dict[str, Any]: | |
| return asdict(self) | |
| def from_dict(cls, data: Dict[str, Any]) -> "DocumentChunk": | |
| return cls(**data) | |
| class ParsedDocument: | |
| doc_id: str | |
| file_name: str | |
| file_type: str | |
| total_pages: int | |
| chunks: List[DocumentChunk] | |
| metadata: Dict[str, Any] = field(default_factory=dict) | |
| def count_tokens(text: str) -> int: | |
| """Accurately count tokens using tiktoken cl100k_base or word fallback.""" | |
| if _TIKTOKEN_AVAILABLE and _ENCODER is not None: | |
| return len(_ENCODER.encode(text, disallowed_special=())) | |
| # Fallback approximation: 1 token ~ 0.75 words / 4 chars | |
| return max(1, int(len(text.split()) * 1.3)) | |
| def table_to_markdown(table_rows: List[List[Optional[str]]]) -> str: | |
| """Converts a 2D list of cells into a clean GitHub-flavored Markdown table.""" | |
| if not table_rows or not any(table_rows): | |
| return "" | |
| cleaned_rows: List[List[str]] = [] | |
| for row in table_rows: | |
| if not row: | |
| continue | |
| cleaned_rows.append([str(c or "").strip().replace("\n", " ") for c in row]) | |
| if not cleaned_rows: | |
| return "" | |
| num_cols = max(len(r) for r in cleaned_rows) | |
| # Pad rows to have uniform column count | |
| for r in cleaned_rows: | |
| while len(r) < num_cols: | |
| r.append("") | |
| header = cleaned_rows[0] | |
| separator = ["---"] * num_cols | |
| body = cleaned_rows[1:] if len(cleaned_rows) > 1 else [] | |
| lines = [ | |
| "| " + " | ".join(header) + " |", | |
| "| " + " | ".join(separator) + " |", | |
| ] | |
| for r in body: | |
| lines.append("| " + " | ".join(r) + " |") | |
| return "\n".join(lines) | |
| class DocumentIngestionPipeline: | |
| def __init__( | |
| self, | |
| chunk_size: int = 400, | |
| chunk_overlap: int = 50, | |
| min_chunk_size: int = 40, | |
| ocr_fallback: bool = True, | |
| ocr_min_chars: int = 40, | |
| ): | |
| self.chunk_size = chunk_size | |
| self.chunk_overlap = chunk_overlap | |
| self.min_chunk_size = min_chunk_size | |
| self.ocr_fallback = ocr_fallback | |
| self.ocr_min_chars = ocr_min_chars | |
| def ingest_file( | |
| self, | |
| file_path: str, | |
| doc_id: Optional[str] = None, | |
| original_filename: Optional[str] = None | |
| ) -> ParsedDocument: | |
| """Main entry point to parse and chunk a document.""" | |
| if not os.path.exists(file_path): | |
| raise FileNotFoundError(f"File not found: {file_path}") | |
| file_name = original_filename or os.path.basename(file_path) | |
| ext = os.path.splitext(file_name)[1].lower() | |
| if not doc_id: | |
| doc_id = str(uuid.uuid4())[:8] | |
| # Ensure working path has proper extension for third-party parsers | |
| actual_path = file_path | |
| temp_target = None | |
| if not file_path.lower().endswith(ext) and ext: | |
| import shutil | |
| temp_target = f"{file_path}_{uuid.uuid4().hex[:6]}{ext}" | |
| shutil.copyfile(file_path, temp_target) | |
| actual_path = temp_target | |
| try: | |
| if ext == ".pdf": | |
| raw_elements, total_pages = self._parse_pdf(actual_path) | |
| file_type = "pdf" | |
| elif ext in [".docx", ".doc"]: | |
| raw_elements, total_pages = self._parse_docx(actual_path) | |
| file_type = "docx" | |
| elif ext in [".txt", ".md"]: | |
| raw_elements, total_pages = self._parse_txt(actual_path) | |
| file_type = "txt" | |
| else: | |
| raise ValueError(f"Unsupported file type: {ext}. Supported: .pdf, .docx, .txt, .md") | |
| # Chunk parsed elements with structure awareness | |
| chunks = self._chunk_elements(raw_elements, doc_id, file_name) | |
| finally: | |
| if temp_target and os.path.exists(temp_target): | |
| try: | |
| os.remove(temp_target) | |
| except Exception: | |
| pass | |
| return ParsedDocument( | |
| doc_id=doc_id, | |
| file_name=file_name, | |
| file_type=file_type, | |
| total_pages=total_pages, | |
| chunks=chunks, | |
| metadata={"file_path": file_path, "total_chunks": len(chunks)} | |
| ) | |
| def _parse_pdf(self, file_path: str) -> Tuple[List[Dict[str, Any]], int]: | |
| """Extract text and tables from PDF, with OCR fallback for scanned pages.""" | |
| elements = [] | |
| total_pages = 0 | |
| # Try pdfplumber first for high fidelity layout & table extraction | |
| if _PDFPLUMBER_AVAILABLE: | |
| with pdfplumber.open(file_path) as pdf: | |
| total_pages = len(pdf.pages) | |
| current_section = "Introduction" | |
| # Pre-load pypdf reader for clean word boundaries and whitespace | |
| pypdf_reader = None | |
| if _PYPDF_AVAILABLE: | |
| try: | |
| pypdf_reader = pypdf.PdfReader(file_path) | |
| except Exception: | |
| pypdf_reader = None | |
| for page_idx, page in enumerate(pdf.pages): | |
| page_num = page_idx + 1 | |
| page_elements = [] | |
| # 1. Extract tables | |
| extracted_tables = [] | |
| try: | |
| tables = page.extract_tables() | |
| for table in tables: | |
| md_table = table_to_markdown(table) | |
| if md_table: | |
| extracted_tables.append(md_table) | |
| except Exception: | |
| pass | |
| # 2. Extract text: prefer pypdf for clean token separation and proper spacing | |
| raw_text = "" | |
| if pypdf_reader is not None and page_idx < len(pypdf_reader.pages): | |
| try: | |
| raw_text = pypdf_reader.pages[page_idx].extract_text() or "" | |
| except Exception: | |
| raw_text = "" | |
| # Fallback to pdfplumber text if pypdf was empty | |
| if not raw_text.strip(): | |
| raw_text = page.extract_text(layout=False) or "" | |
| # 3. OCR Fallback for scanned/empty pages | |
| is_ocr = False | |
| if len(raw_text.strip()) < self.ocr_min_chars and self.ocr_fallback and _OCR_AVAILABLE: | |
| try: | |
| # Convert single page to image and OCR | |
| images = convert_from_path( | |
| file_path, | |
| first_page=page_num, | |
| last_page=page_num, | |
| dpi=200 | |
| ) | |
| if images: | |
| ocr_text = pytesseract.image_to_string(images[0]) | |
| if len(ocr_text.strip()) > len(raw_text.strip()): | |
| raw_text = ocr_text | |
| is_ocr = True | |
| except Exception: | |
| # If OCR tools (e.g. tesseract/poppler) fail, keep original text | |
| pass | |
| # Process paragraphs and detect headings | |
| lines = raw_text.splitlines() | |
| para_buf = [] | |
| for line in lines: | |
| clean_line = line.strip() | |
| if not clean_line: | |
| if para_buf: | |
| para_text = " ".join(para_buf) | |
| page_elements.append({ | |
| "text": para_text, | |
| "page": page_num, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": is_ocr | |
| }) | |
| para_buf = [] | |
| continue | |
| # Heading heuristics: Short, title-case/all-caps, or numbered | |
| is_heading = False | |
| if ( | |
| (len(clean_line) < 70 and not clean_line.endswith((".", ","))) | |
| and ( | |
| clean_line.isupper() | |
| or re.match(r"^(Chapter|\d+(\.\d+)*)\b", clean_line, re.IGNORECASE) | |
| or clean_line.startswith("#") | |
| ) | |
| ): | |
| is_heading = True | |
| if is_heading: | |
| if para_buf: | |
| para_text = " ".join(para_buf) | |
| page_elements.append({ | |
| "text": para_text, | |
| "page": page_num, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": is_ocr | |
| }) | |
| para_buf = [] | |
| current_section = clean_line.lstrip("#").strip() | |
| else: | |
| para_buf.append(clean_line) | |
| if para_buf: | |
| page_elements.append({ | |
| "text": " ".join(para_buf), | |
| "page": page_num, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": is_ocr | |
| }) | |
| # Add extracted tables | |
| for tbl in extracted_tables: | |
| page_elements.append({ | |
| "text": tbl, | |
| "page": page_num, | |
| "section": current_section, | |
| "is_table": True, | |
| "is_ocr": is_ocr | |
| }) | |
| elements.extend(page_elements) | |
| elif _PYPDF_AVAILABLE: | |
| reader = pypdf.PdfReader(file_path) | |
| total_pages = len(reader.pages) | |
| current_section = "General" | |
| for idx, page in enumerate(reader.pages): | |
| page_num = idx + 1 | |
| text = page.extract_text() or "" | |
| elements.append({ | |
| "text": text, | |
| "page": page_num, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": False | |
| }) | |
| else: | |
| raise ImportError("Neither pdfplumber nor pypdf is installed for PDF parsing.") | |
| return elements, total_pages | |
| def _parse_docx(self, file_path: str) -> Tuple[List[Dict[str, Any]], int]: | |
| """Extract paragraphs, headings, and tables from DOCX with approximate page indexing.""" | |
| if not _DOCX_AVAILABLE: | |
| raise ImportError("python-docx is not installed.") | |
| doc = docx.Document(file_path) | |
| elements = [] | |
| current_section = "Overview" | |
| current_page = 1 | |
| words_on_page = 0 | |
| WORDS_PER_PAGE = 350 # Average words per standard page | |
| for element in doc.element.body: | |
| # Check if paragraph | |
| if element.tag.endswith("p"): | |
| # Find matching paragraph object | |
| p_text = "" | |
| p_style = "" | |
| for p in doc.paragraphs: | |
| if p._p == element: | |
| p_text = p.text.strip() | |
| p_style = p.style.name if p.style else "" | |
| break | |
| if not p_text: | |
| continue | |
| if "Heading" in p_style or "Title" in p_style: | |
| current_section = p_text | |
| continue | |
| w_count = len(p_text.split()) | |
| words_on_page += w_count | |
| if words_on_page > WORDS_PER_PAGE: | |
| current_page += 1 | |
| words_on_page = w_count | |
| elements.append({ | |
| "text": p_text, | |
| "page": current_page, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": False | |
| }) | |
| # Check if table | |
| elif element.tag.endswith("tbl"): | |
| for t in doc.tables: | |
| if t._tbl == element: | |
| table_data = [] | |
| for row in t.rows: | |
| table_data.append([cell.text.strip() for cell in row.cells]) | |
| md_table = table_to_markdown(table_data) | |
| if md_table: | |
| elements.append({ | |
| "text": md_table, | |
| "page": current_page, | |
| "section": current_section, | |
| "is_table": True, | |
| "is_ocr": False | |
| }) | |
| break | |
| total_pages = max(current_page, 1) | |
| return elements, total_pages | |
| def _parse_txt(self, file_path: str) -> Tuple[List[Dict[str, Any]], int]: | |
| """Extract text and markdown-style headings from plain text or markdown.""" | |
| with open(file_path, "r", encoding="utf-8", errors="replace") as f: | |
| content = f.read() | |
| lines = content.splitlines() | |
| elements = [] | |
| current_section = "General" | |
| current_page = 1 | |
| para_buf = [] | |
| words_on_page = 0 | |
| WORDS_PER_PAGE = 350 | |
| for line in lines: | |
| clean = line.strip() | |
| if not clean: | |
| if para_buf: | |
| text_block = " ".join(para_buf) | |
| w_count = len(text_block.split()) | |
| words_on_page += w_count | |
| if words_on_page > WORDS_PER_PAGE: | |
| current_page += 1 | |
| words_on_page = w_count | |
| elements.append({ | |
| "text": text_block, | |
| "page": current_page, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": False | |
| }) | |
| para_buf = [] | |
| continue | |
| # Heading detection | |
| if clean.startswith("#"): | |
| if para_buf: | |
| elements.append({ | |
| "text": " ".join(para_buf), | |
| "page": current_page, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": False | |
| }) | |
| para_buf = [] | |
| current_section = clean.lstrip("#").strip() | |
| elif clean.isupper() and len(clean) < 60: | |
| if para_buf: | |
| elements.append({ | |
| "text": " ".join(para_buf), | |
| "page": current_page, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": False | |
| }) | |
| para_buf = [] | |
| current_section = clean | |
| else: | |
| para_buf.append(clean) | |
| if para_buf: | |
| elements.append({ | |
| "text": " ".join(para_buf), | |
| "page": current_page, | |
| "section": current_section, | |
| "is_table": False, | |
| "is_ocr": False | |
| }) | |
| total_pages = max(current_page, 1) | |
| return elements, total_pages | |
| def _chunk_elements( | |
| self, | |
| raw_elements: List[Dict[str, Any]], | |
| doc_id: str, | |
| file_name: str | |
| ) -> List[DocumentChunk]: | |
| """ | |
| Structure-aware semantic chunking. | |
| - Keeps page number and section title as metadata. | |
| - Respects table boundaries (does not slice small tables). | |
| - Target chunk size: 300 - 500 tokens. | |
| - Overlap: 10 - 15% (e.g. 50 tokens). | |
| """ | |
| chunks: List[DocumentChunk] = [] | |
| chunk_idx = 0 | |
| # Group elements by (section, page) when logical, or accumulate up to chunk_size | |
| current_tokens: List[str] = [] | |
| current_page = 1 | |
| current_section = "General" | |
| is_table = False | |
| is_ocr = False | |
| for elem in raw_elements: | |
| elem_text = elem["text"].strip() | |
| if not elem_text: | |
| continue | |
| elem_page = elem["page"] | |
| elem_section = elem["section"] | |
| elem_is_table = elem.get("is_table", False) | |
| elem_is_ocr = elem.get("is_ocr", False) | |
| # If element is a table and fits in chunk_size, keep as dedicated chunk | |
| elem_tokens_count = count_tokens(elem_text) | |
| if elem_is_table and elem_tokens_count <= self.chunk_size: | |
| # Flush existing buffer first | |
| if current_tokens: | |
| c_text = " ".join(current_tokens) | |
| if c_text.strip(): | |
| chunk_id = f"{doc_id}_{chunk_idx:04d}" | |
| chunks.append(DocumentChunk( | |
| chunk_id=chunk_id, | |
| doc_id=doc_id, | |
| file_name=file_name, | |
| text=c_text, | |
| page_number=current_page, | |
| section_title=current_section, | |
| chunk_index=chunk_idx, | |
| token_count=count_tokens(c_text), | |
| metadata={"is_table": is_table, "is_ocr": is_ocr} | |
| )) | |
| chunk_idx += 1 | |
| current_tokens = [] | |
| # Emit table chunk | |
| chunk_id = f"{doc_id}_{chunk_idx:04d}" | |
| chunks.append(DocumentChunk( | |
| chunk_id=chunk_id, | |
| doc_id=doc_id, | |
| file_name=file_name, | |
| text=elem_text, | |
| page_number=elem_page, | |
| section_title=elem_section, | |
| chunk_index=chunk_idx, | |
| token_count=elem_tokens_count, | |
| metadata={"is_table": True, "is_ocr": elem_is_ocr} | |
| )) | |
| chunk_idx += 1 | |
| current_page = elem_page | |
| current_section = elem_section | |
| continue | |
| # If section changed and we have existing tokens, flush current chunk | |
| if elem_section != current_section and len(current_tokens) > 0: | |
| c_text = " ".join(current_tokens) | |
| if c_text.strip(): | |
| chunk_id = f"{doc_id}_{chunk_idx:04d}" | |
| chunks.append(DocumentChunk( | |
| chunk_id=chunk_id, | |
| doc_id=doc_id, | |
| file_name=file_name, | |
| text=c_text, | |
| page_number=current_page, | |
| section_title=current_section, | |
| chunk_index=chunk_idx, | |
| token_count=count_tokens(c_text), | |
| metadata={"is_table": is_table, "is_ocr": is_ocr} | |
| )) | |
| chunk_idx += 1 | |
| current_tokens = [] | |
| # Sentence/paragraph level splitting to build chunks | |
| sentences = re.split(r"(?<=[.!?])\s+", elem_text) | |
| for s in sentences: | |
| s = s.strip() | |
| if not s: | |
| continue | |
| # Check candidate size | |
| test_text = " ".join(current_tokens + [s]) | |
| test_token_count = count_tokens(test_text) | |
| if test_token_count <= self.chunk_size: | |
| current_tokens.append(s) | |
| current_page = elem_page | |
| current_section = elem_section | |
| is_ocr = elem_is_ocr or is_ocr | |
| else: | |
| # Flush current chunk | |
| if current_tokens: | |
| c_text = " ".join(current_tokens) | |
| if count_tokens(c_text) >= self.min_chunk_size: | |
| chunk_id = f"{doc_id}_{chunk_idx:04d}" | |
| chunks.append(DocumentChunk( | |
| chunk_id=chunk_id, | |
| doc_id=doc_id, | |
| file_name=file_name, | |
| text=c_text, | |
| page_number=current_page, | |
| section_title=current_section, | |
| chunk_index=chunk_idx, | |
| token_count=count_tokens(c_text), | |
| metadata={"is_table": is_table, "is_ocr": is_ocr} | |
| )) | |
| chunk_idx += 1 | |
| # Compute overlap tokens from end of previous chunk | |
| overlap_tokens: List[str] = [] | |
| overlap_count = 0 | |
| for prev_s in reversed(current_tokens): | |
| s_tokens = count_tokens(prev_s) | |
| if overlap_count + s_tokens <= self.chunk_overlap: | |
| overlap_tokens.insert(0, prev_s) | |
| overlap_count += s_tokens | |
| else: | |
| break | |
| current_tokens = overlap_tokens + [s] | |
| else: | |
| current_tokens = [s] | |
| current_page = elem_page | |
| current_section = elem_section | |
| is_ocr = elem_is_ocr | |
| # Flush final remaining tokens | |
| if current_tokens: | |
| c_text = " ".join(current_tokens) | |
| if c_text.strip(): | |
| chunk_id = f"{doc_id}_{chunk_idx:04d}" | |
| chunks.append(DocumentChunk( | |
| chunk_id=chunk_id, | |
| doc_id=doc_id, | |
| file_name=file_name, | |
| text=c_text, | |
| page_number=current_page, | |
| section_title=current_section, | |
| chunk_index=chunk_idx, | |
| token_count=count_tokens(c_text), | |
| metadata={"is_table": is_table, "is_ocr": is_ocr} | |
| )) | |
| return chunks | |