Download documentProcessor.py from Ayush-003/ThinkOV2: direct link, hf CLI and curl.
- Browser
- Download file 3.27 kB
-
https://huggingface.co/spaces/Ayush-003/ThinkOV2/resolve/main/documentProcessor.py
- Command line
-
hf download hf://spaces/Ayush-003/ThinkOV2/documentProcessor.py
-
curl -L -o documentProcessor.py https://huggingface.co/spaces/Ayush-003/ThinkOV2/resolve/main/documentProcessor.py
3.27 kB
| """ | |
| Enhanced document processor with multiple format support | |
| """ | |
| from langchain_community.document_loaders import TextLoader | |
| from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter | |
| from langchain.schema import Document | |
| from typing import List, Optional | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| class DocumentProcessor: | |
| """Enhanced document processor""" | |
| def __init__(self, file_path: Optional[str] = None, chunk_size: int = 1000, chunk_overlap: int = 200): | |
| self.file_path = file_path | |
| self.chunk_size = chunk_size | |
| self.chunk_overlap = chunk_overlap | |
| def load_and_split(self) -> List[Document]: | |
| """Load document from file and split into chunks""" | |
| if not self.file_path: | |
| raise ValueError("file_path must be provided") | |
| try: | |
| loader = TextLoader(self.file_path, encoding='utf-8') | |
| documents = loader.load() | |
| # Use recursive splitter for better chunking | |
| splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=self.chunk_size, | |
| chunk_overlap=self.chunk_overlap, | |
| separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""], | |
| length_function=len | |
| ) | |
| texts = splitter.split_documents(documents) | |
| logger.info(f"Processed {len(texts)} chunks from {self.file_path}") | |
| return texts | |
| except Exception as e: | |
| logger.error(f"Error processing document: {e}") | |
| raise | |
| def split_text(self, text: str, metadata: Optional[dict] = None) -> List[Document]: | |
| """Split raw text into chunks""" | |
| try: | |
| splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=self.chunk_size, | |
| chunk_overlap=self.chunk_overlap, | |
| separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""], | |
| length_function=len | |
| ) | |
| # Create document | |
| doc = Document(page_content=text, metadata=metadata or {}) | |
| texts = splitter.split_documents([doc]) | |
| logger.info(f"Split text into {len(texts)} chunks") | |
| return texts | |
| except Exception as e: | |
| logger.error(f"Error splitting text: {e}") | |
| raise | |
| def create_documents_from_texts( | |
| self, | |
| texts: List[str], | |
| metadatas: Optional[List[dict]] = None | |
| ) -> List[Document]: | |
| """Create Document objects from list of texts""" | |
| documents = [] | |
| for i, text in enumerate(texts): | |
| metadata = metadatas[i] if metadatas and i < len(metadatas) else {} | |
| doc = Document(page_content=text, metadata=metadata) | |
| documents.append(doc) | |
| return documents | |
| if __name__ == "__main__": | |
| processor = DocumentProcessor("ecommerce_faq.txt") | |
| texts = processor.load_and_split() | |
| print(f"Processed {len(texts)} text chunks") | |
| # Print first chunk | |
| if texts: | |
| print(f"\nFirst chunk:\n{texts[0].page_content[:200]}...") | |