import logging from typing import List from pathlib import Path import docx from docx.document import Document as _Document from docx.oxml.text.paragraph import CT_P from docx.oxml.table import CT_Tbl from docx.table import _Cell, Table from docx.text.paragraph import Paragraph import pandas as pd import pypdf from langchain.docstore.document import Document from langchain.text_splitter import RecursiveCharacterTextSplitter from app.core.config import settings from unstructured.partition.auto import partition logger = logging.getLogger(__name__) class DocumentProcessor: def __init__(self): self.text_splitter = RecursiveCharacterTextSplitter( chunk_size=settings.chunk_size, chunk_overlap=settings.chunk_overlap, separators=["\n\n", "\n", " ", ""] ) def process_directory(self, directory_path: Path) -> List[Document]: documents = [] if not directory_path.exists(): logger.warning(f"Directory not found: {directory_path}") return [] # Varre recursivamente todas as subpastas for file_path in directory_path.rglob("*"): if file_path.is_file() and file_path.suffix.lower() in settings.supported_extensions: try: # Determina o escopo baseado na pasta pai imediata dentro de raw_documents # Se estiver na raiz, escopo é 'default' relative_path = file_path.relative_to(directory_path) if len(relative_path.parts) > 1: scope = relative_path.parts[0] # A primeira pasta é o escopo else: scope = "default" docs = self.process_file(file_path, scope) documents.extend(docs) except Exception as e: logger.error(f"Error processing file {file_path}: {e}") return documents def _add_chunk_metadata(self, chunks: List[Document], scope: str) -> List[Document]: for i, chunk in enumerate(chunks): chunk.metadata["chunk"] = i chunk.metadata["scope"] = scope # Adiciona o escopo aos metadados return chunks def process_file(self, file_path: Path, scope: str = "default") -> List[Document]: logger.info(f"Processing file: {file_path} (Scope: {scope})") text_content = "" try: file_suffix = file_path.suffix.lower() if file_suffix == '.docx': return self._process_docx_ordered(file_path, scope) elif file_suffix == '.pdf': with open(file_path, 'rb') as f: reader = pypdf.PdfReader(f) all_text = [page.extract_text() for page in reader.pages if page.extract_text()] text_content = "\n\n".join(all_text) elif file_suffix in ['.xls', '.xlsx']: try: df_dict = pd.read_excel(file_path, sheet_name=None) all_text = [] for sheet_name, df in df_dict.items(): all_text.append(f"--- Sheet: {sheet_name} ---\n") all_text.append(df.to_string(index=False)) text_content = "\n\n".join(all_text) except Exception as e: logger.error(f"Pandas failed for {file_path}: {e}") return [] elif file_suffix == '.ini': text_content = file_path.read_text(encoding='utf-8', errors='ignore') doc = Document( page_content=text_content, metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix, "chunk": 0} ) return self._add_chunk_metadata([doc], scope) else: elements = partition(filename=str(file_path)) text_content = "\n\n".join([str(el) for el in elements]) if not text_content.strip(): logger.warning(f"File {file_path} yielded no text content.") return [] doc = Document( page_content=text_content, metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix} ) chunks = self.text_splitter.split_documents([doc]) return self._add_chunk_metadata(chunks, scope) except Exception as e: logger.error(f"Failed to process {file_path}: {e}") return [] def _process_docx_ordered(self, file_path: Path, scope: str) -> List[Document]: try: doc = docx.Document(str(file_path)) full_text_parts = [] for element in doc.element.body: if isinstance(element, CT_P): para = Paragraph(element, doc) if para.text.strip(): full_text_parts.append(para.text.strip()) elif isinstance(element, CT_Tbl): table = Table(element, doc) try: rows_md = [] if len(table.rows) > 0: header_cells = [cell.text.strip().replace('\n', ' ') for cell in table.rows[0].cells] rows_md.append("| " + " | ".join(header_cells) + " |") rows_md.append("| " + " | ".join(["---"] * len(header_cells)) + " |") for row in table.rows[1:]: row_cells = [cell.text.strip().replace('\n', ' ') for cell in row.cells] rows_md.append("| " + " | ".join(row_cells) + " |") full_text_parts.append("\n" + "\n".join(rows_md) + "\n") except Exception as table_e: logger.warning(f"Error parsing table in {file_path.name}: {table_e}") continue full_text = "\n\n".join(full_text_parts) if not full_text.strip(): return [] doc_obj = Document( page_content=full_text, metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix} ) chunks = self.text_splitter.split_documents([doc_obj]) return self._add_chunk_metadata(chunks, scope) except Exception as e: logger.error(f"Error in _process_docx_ordered for {file_path}: {e}") return []