File size: 6,745 Bytes
7dad342 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | import logging
from typing import List
from pathlib import Path
import docx
from docx.document import Document as _Document
from docx.oxml.text.paragraph import CT_P
from docx.oxml.table import CT_Tbl
from docx.table import _Cell, Table
from docx.text.paragraph import Paragraph
import pandas as pd
import pypdf
from langchain.docstore.document import Document
from langchain.text_splitter import RecursiveCharacterTextSplitter
from app.core.config import settings
from unstructured.partition.auto import partition
logger = logging.getLogger(__name__)
class DocumentProcessor:
def __init__(self):
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=settings.chunk_size,
chunk_overlap=settings.chunk_overlap,
separators=["\n\n", "\n", " ", ""]
)
def process_directory(self, directory_path: Path) -> List[Document]:
documents = []
if not directory_path.exists():
logger.warning(f"Directory not found: {directory_path}")
return []
# Varre recursivamente todas as subpastas
for file_path in directory_path.rglob("*"):
if file_path.is_file() and file_path.suffix.lower() in settings.supported_extensions:
try:
# Determina o escopo baseado na pasta pai imediata dentro de raw_documents
# Se estiver na raiz, escopo é 'default'
relative_path = file_path.relative_to(directory_path)
if len(relative_path.parts) > 1:
scope = relative_path.parts[0] # A primeira pasta é o escopo
else:
scope = "default"
docs = self.process_file(file_path, scope)
documents.extend(docs)
except Exception as e:
logger.error(f"Error processing file {file_path}: {e}")
return documents
def _add_chunk_metadata(self, chunks: List[Document], scope: str) -> List[Document]:
for i, chunk in enumerate(chunks):
chunk.metadata["chunk"] = i
chunk.metadata["scope"] = scope # Adiciona o escopo aos metadados
return chunks
def process_file(self, file_path: Path, scope: str = "default") -> List[Document]:
logger.info(f"Processing file: {file_path} (Scope: {scope})")
text_content = ""
try:
file_suffix = file_path.suffix.lower()
if file_suffix == '.docx':
return self._process_docx_ordered(file_path, scope)
elif file_suffix == '.pdf':
with open(file_path, 'rb') as f:
reader = pypdf.PdfReader(f)
all_text = [page.extract_text() for page in reader.pages if page.extract_text()]
text_content = "\n\n".join(all_text)
elif file_suffix in ['.xls', '.xlsx']:
try:
df_dict = pd.read_excel(file_path, sheet_name=None)
all_text = []
for sheet_name, df in df_dict.items():
all_text.append(f"--- Sheet: {sheet_name} ---\n")
all_text.append(df.to_string(index=False))
text_content = "\n\n".join(all_text)
except Exception as e:
logger.error(f"Pandas failed for {file_path}: {e}")
return []
elif file_suffix == '.ini':
text_content = file_path.read_text(encoding='utf-8', errors='ignore')
doc = Document(
page_content=text_content,
metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix, "chunk": 0}
)
return self._add_chunk_metadata([doc], scope)
else:
elements = partition(filename=str(file_path))
text_content = "\n\n".join([str(el) for el in elements])
if not text_content.strip():
logger.warning(f"File {file_path} yielded no text content.")
return []
doc = Document(
page_content=text_content,
metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix}
)
chunks = self.text_splitter.split_documents([doc])
return self._add_chunk_metadata(chunks, scope)
except Exception as e:
logger.error(f"Failed to process {file_path}: {e}")
return []
def _process_docx_ordered(self, file_path: Path, scope: str) -> List[Document]:
try:
doc = docx.Document(str(file_path))
full_text_parts = []
for element in doc.element.body:
if isinstance(element, CT_P):
para = Paragraph(element, doc)
if para.text.strip():
full_text_parts.append(para.text.strip())
elif isinstance(element, CT_Tbl):
table = Table(element, doc)
try:
rows_md = []
if len(table.rows) > 0:
header_cells = [cell.text.strip().replace('\n', ' ') for cell in table.rows[0].cells]
rows_md.append("| " + " | ".join(header_cells) + " |")
rows_md.append("| " + " | ".join(["---"] * len(header_cells)) + " |")
for row in table.rows[1:]:
row_cells = [cell.text.strip().replace('\n', ' ') for cell in row.cells]
rows_md.append("| " + " | ".join(row_cells) + " |")
full_text_parts.append("\n" + "\n".join(rows_md) + "\n")
except Exception as table_e:
logger.warning(f"Error parsing table in {file_path.name}: {table_e}")
continue
full_text = "\n\n".join(full_text_parts)
if not full_text.strip():
return []
doc_obj = Document(
page_content=full_text,
metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix}
)
chunks = self.text_splitter.split_documents([doc_obj])
return self._add_chunk_metadata(chunks, scope)
except Exception as e:
logger.error(f"Error in _process_docx_ordered for {file_path}: {e}")
return []
|