Download app/services/document_processor.py from aionaion/chatdoc: direct link, hf CLI and curl.
- Browser
- Download file 6.75 kB
-
https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/app/services/document_processor.py
- Command line
-
hf download hf://spaces/aionaion/chatdoc/app/services/document_processor.py
-
curl -L -o document_processor.py https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/app/services/document_processor.py
6.75 kB
| import logging | |
| from typing import List | |
| from pathlib import Path | |
| import docx | |
| from docx.document import Document as _Document | |
| from docx.oxml.text.paragraph import CT_P | |
| from docx.oxml.table import CT_Tbl | |
| from docx.table import _Cell, Table | |
| from docx.text.paragraph import Paragraph | |
| import pandas as pd | |
| import pypdf | |
| from langchain.docstore.document import Document | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from app.core.config import settings | |
| from unstructured.partition.auto import partition | |
| logger = logging.getLogger(__name__) | |
| class DocumentProcessor: | |
| def __init__(self): | |
| self.text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=settings.chunk_size, | |
| chunk_overlap=settings.chunk_overlap, | |
| separators=["\n\n", "\n", " ", ""] | |
| ) | |
| def process_directory(self, directory_path: Path) -> List[Document]: | |
| documents = [] | |
| if not directory_path.exists(): | |
| logger.warning(f"Directory not found: {directory_path}") | |
| return [] | |
| # Varre recursivamente todas as subpastas | |
| for file_path in directory_path.rglob("*"): | |
| if file_path.is_file() and file_path.suffix.lower() in settings.supported_extensions: | |
| try: | |
| # Determina o escopo baseado na pasta pai imediata dentro de raw_documents | |
| # Se estiver na raiz, escopo é 'default' | |
| relative_path = file_path.relative_to(directory_path) | |
| if len(relative_path.parts) > 1: | |
| scope = relative_path.parts[0] # A primeira pasta é o escopo | |
| else: | |
| scope = "default" | |
| docs = self.process_file(file_path, scope) | |
| documents.extend(docs) | |
| except Exception as e: | |
| logger.error(f"Error processing file {file_path}: {e}") | |
| return documents | |
| def _add_chunk_metadata(self, chunks: List[Document], scope: str) -> List[Document]: | |
| for i, chunk in enumerate(chunks): | |
| chunk.metadata["chunk"] = i | |
| chunk.metadata["scope"] = scope # Adiciona o escopo aos metadados | |
| return chunks | |
| def process_file(self, file_path: Path, scope: str = "default") -> List[Document]: | |
| logger.info(f"Processing file: {file_path} (Scope: {scope})") | |
| text_content = "" | |
| try: | |
| file_suffix = file_path.suffix.lower() | |
| if file_suffix == '.docx': | |
| return self._process_docx_ordered(file_path, scope) | |
| elif file_suffix == '.pdf': | |
| with open(file_path, 'rb') as f: | |
| reader = pypdf.PdfReader(f) | |
| all_text = [page.extract_text() for page in reader.pages if page.extract_text()] | |
| text_content = "\n\n".join(all_text) | |
| elif file_suffix in ['.xls', '.xlsx']: | |
| try: | |
| df_dict = pd.read_excel(file_path, sheet_name=None) | |
| all_text = [] | |
| for sheet_name, df in df_dict.items(): | |
| all_text.append(f"--- Sheet: {sheet_name} ---\n") | |
| all_text.append(df.to_string(index=False)) | |
| text_content = "\n\n".join(all_text) | |
| except Exception as e: | |
| logger.error(f"Pandas failed for {file_path}: {e}") | |
| return [] | |
| elif file_suffix == '.ini': | |
| text_content = file_path.read_text(encoding='utf-8', errors='ignore') | |
| doc = Document( | |
| page_content=text_content, | |
| metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix, "chunk": 0} | |
| ) | |
| return self._add_chunk_metadata([doc], scope) | |
| else: | |
| elements = partition(filename=str(file_path)) | |
| text_content = "\n\n".join([str(el) for el in elements]) | |
| if not text_content.strip(): | |
| logger.warning(f"File {file_path} yielded no text content.") | |
| return [] | |
| doc = Document( | |
| page_content=text_content, | |
| metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix} | |
| ) | |
| chunks = self.text_splitter.split_documents([doc]) | |
| return self._add_chunk_metadata(chunks, scope) | |
| except Exception as e: | |
| logger.error(f"Failed to process {file_path}: {e}") | |
| return [] | |
| def _process_docx_ordered(self, file_path: Path, scope: str) -> List[Document]: | |
| try: | |
| doc = docx.Document(str(file_path)) | |
| full_text_parts = [] | |
| for element in doc.element.body: | |
| if isinstance(element, CT_P): | |
| para = Paragraph(element, doc) | |
| if para.text.strip(): | |
| full_text_parts.append(para.text.strip()) | |
| elif isinstance(element, CT_Tbl): | |
| table = Table(element, doc) | |
| try: | |
| rows_md = [] | |
| if len(table.rows) > 0: | |
| header_cells = [cell.text.strip().replace('\n', ' ') for cell in table.rows[0].cells] | |
| rows_md.append("| " + " | ".join(header_cells) + " |") | |
| rows_md.append("| " + " | ".join(["---"] * len(header_cells)) + " |") | |
| for row in table.rows[1:]: | |
| row_cells = [cell.text.strip().replace('\n', ' ') for cell in row.cells] | |
| rows_md.append("| " + " | ".join(row_cells) + " |") | |
| full_text_parts.append("\n" + "\n".join(rows_md) + "\n") | |
| except Exception as table_e: | |
| logger.warning(f"Error parsing table in {file_path.name}: {table_e}") | |
| continue | |
| full_text = "\n\n".join(full_text_parts) | |
| if not full_text.strip(): | |
| return [] | |
| doc_obj = Document( | |
| page_content=full_text, | |
| metadata={"source": str(file_path), "filename": file_path.name, "extension": file_path.suffix} | |
| ) | |
| chunks = self.text_splitter.split_documents([doc_obj]) | |
| return self._add_chunk_metadata(chunks, scope) | |
| except Exception as e: | |
| logger.error(f"Error in _process_docx_ordered for {file_path}: {e}") | |
| return [] | |