File size: 6,745 Bytes
7dad342
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
import logging
from typing import List
from pathlib import Path
import docx
from docx.document import Document as _Document
from docx.oxml.text.paragraph import CT_P
from docx.oxml.table import CT_Tbl
from docx.table import _Cell, Table
from docx.text.paragraph import Paragraph
import pandas as pd
import pypdf
from langchain.docstore.document import Document
from langchain.text_splitter import RecursiveCharacterTextSplitter
from app.core.config import settings
from unstructured.partition.auto import partition

logger = logging.getLogger(__name__)

class DocumentProcessor:
    def __init__(self):
        self.text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=settings.chunk_size,
            chunk_overlap=settings.chunk_overlap,
            separators=["\n\n", "\n", " ", ""]
        )

    def process_directory(self, directory_path: Path) -> List[Document]:
        documents = []
        if not directory_path.exists():
            logger.warning(f"Directory not found: {directory_path}")
            return []

        # Varre recursivamente todas as subpastas
        for file_path in directory_path.rglob("*"):
            if file_path.is_file() and file_path.suffix.lower() in settings.supported_extensions:
                try:
                    # Determina o escopo baseado na pasta pai imediata dentro de raw_documents
                    # Se estiver na raiz, escopo é 'default'
                    relative_path = file_path.relative_to(directory_path)
                    if len(relative_path.parts) > 1:
                        scope = relative_path.parts[0] # A primeira pasta é o escopo
                    else:
                        scope = "default"

                    docs = self.process_file(file_path, scope)
                    documents.extend(docs)
                except Exception as e:
                    logger.error(f"Error processing file {file_path}: {e}")
        return documents

    def _add_chunk_metadata(self, chunks: List[Document], scope: str) -> List[Document]:
        for i, chunk in enumerate(chunks):
            chunk.metadata["chunk"] = i
            chunk.metadata["scope"] = scope # Adiciona o escopo aos metadados
        return chunks

    def process_file(self, file_path: Path, scope: str = "default") -> List[Document]:
        logger.info(f"Processing file: {file_path} (Scope: {scope})")
        text_content = ""
        try:
            file_suffix = file_path.suffix.lower()
            
            if file_suffix == '.docx':
                return self._process_docx_ordered(file_path, scope)
            
            elif file_suffix == '.pdf':
                with open(file_path, 'rb') as f:
                    reader = pypdf.PdfReader(f)
                    all_text = [page.extract_text() for page in reader.pages if page.extract_text()]
                    text_content = "\n\n".join(all_text)

            elif file_suffix in ['.xls', '.xlsx']:
                try:
                    df_dict = pd.read_excel(file_path, sheet_name=None)
                    all_text = []
                    for sheet_name, df in df_dict.items():
                        all_text.append(f"--- Sheet: {sheet_name} ---\n")
                        all_text.append(df.to_string(index=False))
                    text_content = "\n\n".join(all_text)
                except Exception as e:
                    logger.error(f"Pandas failed for {file_path}: {e}")
                    return []
            
            elif file_suffix == '.ini':
                text_content = file_path.read_text(encoding='utf-8', errors='ignore')
                doc = Document(
                    page_content=text_content,
                    metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix, "chunk": 0}
                )
                return self._add_chunk_metadata([doc], scope)

            else: 
                elements = partition(filename=str(file_path))
                text_content = "\n\n".join([str(el) for el in elements])

            if not text_content.strip():
                logger.warning(f"File {file_path} yielded no text content.")
                return []

            doc = Document(
                page_content=text_content,
                metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix}
            )
            chunks = self.text_splitter.split_documents([doc])
            return self._add_chunk_metadata(chunks, scope)

        except Exception as e:
            logger.error(f"Failed to process {file_path}: {e}")
            return []

    def _process_docx_ordered(self, file_path: Path, scope: str) -> List[Document]:
        try:
            doc = docx.Document(str(file_path))
            full_text_parts = []

            for element in doc.element.body:
                if isinstance(element, CT_P):
                    para = Paragraph(element, doc)
                    if para.text.strip():
                        full_text_parts.append(para.text.strip())
                
                elif isinstance(element, CT_Tbl):
                    table = Table(element, doc)
                    try:
                        rows_md = []
                        if len(table.rows) > 0:
                            header_cells = [cell.text.strip().replace('\n', ' ') for cell in table.rows[0].cells]
                            rows_md.append("| " + " | ".join(header_cells) + " |")
                            rows_md.append("| " + " | ".join(["---"] * len(header_cells)) + " |")
                            
                            for row in table.rows[1:]:
                                row_cells = [cell.text.strip().replace('\n', ' ') for cell in row.cells]
                                rows_md.append("| " + " | ".join(row_cells) + " |")
                            
                            full_text_parts.append("\n" + "\n".join(rows_md) + "\n")
                    except Exception as table_e:
                        logger.warning(f"Error parsing table in {file_path.name}: {table_e}")
                        continue

            full_text = "\n\n".join(full_text_parts)

            if not full_text.strip():
                return []

            doc_obj = Document(
                page_content=full_text,
                metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix}
            )
            
            chunks = self.text_splitter.split_documents([doc_obj])
            return self._add_chunk_metadata(chunks, scope)

        except Exception as e:
            logger.error(f"Error in _process_docx_ordered for {file_path}: {e}")
            return []