chatdoc / app\services\document_processor.py
celiolr's picture
Deploy Docker: app\services\document_processor.py
7dad342 verified
Raw History Blame Contribute Delete
6.75 kB
import logging
from typing import List
from pathlib import Path
import docx
from docx.document import Document as _Document
from docx.oxml.text.paragraph import CT_P
from docx.oxml.table import CT_Tbl
from docx.table import _Cell, Table
from docx.text.paragraph import Paragraph
import pandas as pd
import pypdf
from langchain.docstore.document import Document
from langchain.text_splitter import RecursiveCharacterTextSplitter
from app.core.config import settings
from unstructured.partition.auto import partition
logger = logging.getLogger(__name__)
class DocumentProcessor:
def __init__(self):
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=settings.chunk_size,
chunk_overlap=settings.chunk_overlap,
separators=["\n\n", "\n", " ", ""]
)
def process_directory(self, directory_path: Path) -> List[Document]:
documents = []
if not directory_path.exists():
logger.warning(f"Directory not found: {directory_path}")
return []
# Varre recursivamente todas as subpastas
for file_path in directory_path.rglob("*"):
if file_path.is_file() and file_path.suffix.lower() in settings.supported_extensions:
try:
# Determina o escopo baseado na pasta pai imediata dentro de raw_documents
# Se estiver na raiz, escopo é 'default'
relative_path = file_path.relative_to(directory_path)
if len(relative_path.parts) > 1:
scope = relative_path.parts[0] # A primeira pasta é o escopo
else:
scope = "default"
docs = self.process_file(file_path, scope)
documents.extend(docs)
except Exception as e:
logger.error(f"Error processing file {file_path}: {e}")
return documents
def _add_chunk_metadata(self, chunks: List[Document], scope: str) -> List[Document]:
for i, chunk in enumerate(chunks):
chunk.metadata["chunk"] = i
chunk.metadata["scope"] = scope # Adiciona o escopo aos metadados
return chunks
def process_file(self, file_path: Path, scope: str = "default") -> List[Document]:
logger.info(f"Processing file: {file_path} (Scope: {scope})")
text_content = ""
try:
file_suffix = file_path.suffix.lower()
if file_suffix == '.docx':
return self._process_docx_ordered(file_path, scope)
elif file_suffix == '.pdf':
with open(file_path, 'rb') as f:
reader = pypdf.PdfReader(f)
all_text = [page.extract_text() for page in reader.pages if page.extract_text()]
text_content = "\n\n".join(all_text)
elif file_suffix in ['.xls', '.xlsx']:
try:
df_dict = pd.read_excel(file_path, sheet_name=None)
all_text = []
for sheet_name, df in df_dict.items():
all_text.append(f"--- Sheet: {sheet_name} ---\n")
all_text.append(df.to_string(index=False))
text_content = "\n\n".join(all_text)
except Exception as e:
logger.error(f"Pandas failed for {file_path}: {e}")
return []
elif file_suffix == '.ini':
text_content = file_path.read_text(encoding='utf-8', errors='ignore')
doc = Document(
page_content=text_content,
metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix, "chunk": 0}
)
return self._add_chunk_metadata([doc], scope)
else:
elements = partition(filename=str(file_path))
text_content = "\n\n".join([str(el) for el in elements])
if not text_content.strip():
logger.warning(f"File {file_path} yielded no text content.")
return []
doc = Document(
page_content=text_content,
metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix}
)
chunks = self.text_splitter.split_documents([doc])
return self._add_chunk_metadata(chunks, scope)
except Exception as e:
logger.error(f"Failed to process {file_path}: {e}")
return []
def _process_docx_ordered(self, file_path: Path, scope: str) -> List[Document]:
try:
doc = docx.Document(str(file_path))
full_text_parts = []
for element in doc.element.body:
if isinstance(element, CT_P):
para = Paragraph(element, doc)
if para.text.strip():
full_text_parts.append(para.text.strip())
elif isinstance(element, CT_Tbl):
table = Table(element, doc)
try:
rows_md = []
if len(table.rows) > 0:
header_cells = [cell.text.strip().replace('\n', ' ') for cell in table.rows[0].cells]
rows_md.append("| " + " | ".join(header_cells) + " |")
rows_md.append("| " + " | ".join(["---"] * len(header_cells)) + " |")
for row in table.rows[1:]:
row_cells = [cell.text.strip().replace('\n', ' ') for cell in row.cells]
rows_md.append("| " + " | ".join(row_cells) + " |")
full_text_parts.append("\n" + "\n".join(rows_md) + "\n")
except Exception as table_e:
logger.warning(f"Error parsing table in {file_path.name}: {table_e}")
continue
full_text = "\n\n".join(full_text_parts)
if not full_text.strip():
return []
doc_obj = Document(
page_content=full_text,
metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix}
)
chunks = self.text_splitter.split_documents([doc_obj])
return self._add_chunk_metadata(chunks, scope)
except Exception as e:
logger.error(f"Error in _process_docx_ordered for {file_path}: {e}")
return []