Spaces:
Runtime error
Runtime error
Download app/services/rag/document_processor.py from junaidte14/aiverse: direct link, hf CLI and curl.
- Browser
- Download file 8.26 kB
-
https://huggingface.co/spaces/junaidte14/aiverse/resolve/main/app/services/rag/document_processor.py
- Command line
-
hf download hf://spaces/junaidte14/aiverse/app/services/rag/document_processor.py
-
curl -L -o document_processor.py https://huggingface.co/spaces/junaidte14/aiverse/resolve/main/app/services/rag/document_processor.py
8.26 kB
| """ | |
| Document Processing Service | |
| Refined for Versana Ecosystem and pgvector JSONB integration | |
| """ | |
| import re | |
| from typing import List, Dict, Any, Optional | |
| from pathlib import Path | |
| import hashlib | |
| import tiktoken | |
| # Optional imports with fallbacks | |
| try: | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter, Language | |
| LANGCHAIN_AVAILABLE = True | |
| except ImportError: | |
| LANGCHAIN_AVAILABLE = False | |
| try: | |
| import PyPDF2 | |
| PDF_AVAILABLE = True | |
| except ImportError: | |
| PDF_AVAILABLE = False | |
| try: | |
| from docx import Document | |
| DOCX_AVAILABLE = True | |
| except ImportError: | |
| DOCX_AVAILABLE = False | |
| class DocumentProcessor: | |
| """ | |
| Process and chunk documents for the Versana RAG pipeline. | |
| Optimized for pgvector JSONB metadata storage[cite: 5, 11]. | |
| """ | |
| def __init__( | |
| self, | |
| chunk_size: int = 500, # Optimized for Llama and PHP/Code context | |
| chunk_overlap: int = 50, | |
| model_name: str = "gpt-3.5-turbo", | |
| ): | |
| self.chunk_size = chunk_size | |
| self.chunk_overlap = chunk_overlap | |
| try: | |
| self.tokenizer = tiktoken.encoding_for_model(model_name) | |
| except Exception: | |
| self.tokenizer = tiktoken.get_encoding("cl100k_base") | |
| def count_tokens(self, text: str) -> int: | |
| """Count tokens in text accurately for the RAG window.""" | |
| return len(self.tokenizer.encode(text)) | |
| def process_file( | |
| self, file_path: str, metadata: Optional[Dict[str, Any]] = None | |
| ) -> List[Dict[str, Any]]: | |
| """ | |
| Process a file and return chunks with structured JSONB-ready metadata[cite: 5, 11]. | |
| """ | |
| path = Path(file_path) | |
| extension = path.suffix.lower() | |
| # Extract text based on file type | |
| if extension == ".pdf": | |
| text = self._extract_pdf(file_path) | |
| elif extension == ".docx": | |
| text = self._extract_docx(file_path) | |
| elif extension == ".md": | |
| text = self._extract_markdown(file_path) | |
| elif extension in [ | |
| ".py", | |
| ".js", | |
| ".jsx", | |
| ".ts", | |
| ".tsx", | |
| ".php", | |
| ".java", | |
| ".cpp", | |
| ".c", | |
| ".go", | |
| ".rb", | |
| ".rs", | |
| ]: | |
| text = self._extract_code(file_path) | |
| elif extension in [".txt", ".json", ".xml", ".html", ".css"]: | |
| text = self._extract_text(file_path) | |
| else: | |
| return [] | |
| if not text: | |
| return [] | |
| language = self._detect_language(extension) | |
| chunks = self._chunk_text(text, language) | |
| documents = [] | |
| for i, chunk in enumerate(chunks): | |
| doc_id = self._generate_id(file_path, i) | |
| # Structured metadata for high-performance JSONB filtering[cite: 5] | |
| doc_metadata = { | |
| "source": str(path), | |
| "filename": path.name, | |
| "extension": extension, | |
| "chunk_index": i, | |
| "total_chunks": len(chunks), | |
| "tokens": self.count_tokens(chunk), | |
| **(metadata or {}), | |
| } | |
| documents.append({"id": doc_id, "text": chunk, "metadata": doc_metadata}) | |
| return documents | |
| def _extract_pdf(self, file_path: str) -> str: | |
| if not PDF_AVAILABLE: | |
| raise ImportError("PyPDF2 not installed.") | |
| try: | |
| with open(file_path, "rb") as file: | |
| pdf_reader = PyPDF2.PdfReader(file) | |
| return "\n\n".join( | |
| page.extract_text() for page in pdf_reader.pages | |
| ).strip() | |
| except Exception as e: | |
| print(f"Error extracting PDF {file_path}: {e}") | |
| return "" | |
| def _extract_docx(self, file_path: str) -> str: | |
| if not DOCX_AVAILABLE: | |
| raise ImportError("python-docx not installed.") | |
| try: | |
| doc = Document(file_path) | |
| return "\n\n".join(para.text for para in doc.paragraphs).strip() | |
| except Exception as e: | |
| print(f"Error extracting DOCX {file_path}: {e}") | |
| return "" | |
| def _extract_markdown(self, file_path: str) -> str: | |
| return self._extract_text(file_path) | |
| def _extract_code(self, file_path: str) -> str: | |
| """Preserves structure for code files.""" | |
| return self._extract_text(file_path) | |
| def _extract_text(self, file_path: str) -> str: | |
| """Safely extract plain text with encoding fallbacks.""" | |
| try: | |
| with open(file_path, "r", encoding="utf-8") as f: | |
| return f.read() | |
| except UnicodeDecodeError: | |
| try: | |
| with open(file_path, "r", encoding="latin-1") as f: | |
| return f.read() | |
| except Exception: | |
| return "" | |
| except Exception: | |
| return "" | |
| def _detect_language(self, extension: str) -> Optional["Language"]: | |
| """Maps extensions to LangChain languages for smarter chunking.""" | |
| if not LANGCHAIN_AVAILABLE: | |
| return None | |
| lang_map = { | |
| ".py": Language.PYTHON, | |
| ".js": Language.JS, | |
| ".jsx": Language.JS, | |
| ".ts": Language.TS, | |
| ".tsx": Language.TS, | |
| ".php": Language.PHP, # Critical for Versana WordPress projects | |
| ".java": Language.JAVA, | |
| ".cpp": Language.CPP, | |
| ".c": Language.C, | |
| ".go": Language.GO, | |
| ".rb": Language.RUBY, | |
| ".rs": Language.RUST, | |
| ".md": Language.MARKDOWN, | |
| ".html": Language.HTML, | |
| } | |
| return lang_map.get(extension) | |
| def _chunk_text( | |
| self, text: str, language: Optional["Language"] = None | |
| ) -> List[str]: | |
| """Intelligent chunking using LangChain splitters[cite: 11].""" | |
| if not LANGCHAIN_AVAILABLE: | |
| return self._simple_chunk(text) | |
| if language: | |
| splitter = RecursiveCharacterTextSplitter.from_language( | |
| language=language, | |
| chunk_size=self.chunk_size, | |
| chunk_overlap=self.chunk_overlap, | |
| ) | |
| else: | |
| splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=self.chunk_size, | |
| chunk_overlap=self.chunk_overlap, | |
| separators=["\n\n", "\n", ". ", " ", ""], | |
| ) | |
| return splitter.split_text(text) | |
| def _simple_chunk(self, text: str) -> List[str]: | |
| """Fallback character-based chunking[cite: 11].""" | |
| chunks = [] | |
| start = 0 | |
| while start < len(text): | |
| end = start + self.chunk_size | |
| chunks.append(text[start:end].strip()) | |
| start = end - self.chunk_overlap | |
| return [c for c in chunks if c] | |
| def _generate_id(self, file_path: str, chunk_index: int) -> str: | |
| """Consistent ID generation for pgvector upserts[cite: 5, 11].""" | |
| source = f"{file_path}_{chunk_index}" | |
| return hashlib.md5(source.encode()).hexdigest() | |
| def process_directory( | |
| self, | |
| directory: str, | |
| file_extensions: Optional[List[str]] = None, | |
| exclude_patterns: Optional[List[str]] = None, | |
| metadata: Optional[Dict[str, Any]] = None, | |
| ) -> List[Dict[str, Any]]: | |
| """ | |
| Process directory and skip build/vendor artifacts[cite: 11]. | |
| """ | |
| path = Path(directory) | |
| if not path.exists(): | |
| raise ValueError(f"Directory not found: {directory}") | |
| exclude_patterns = exclude_patterns or [ | |
| "__pycache__", | |
| "node_modules", | |
| ".git", | |
| "vendor", | |
| "dist", | |
| "build", | |
| ] | |
| all_documents = [] | |
| for file_path in path.rglob("*"): | |
| if file_path.is_dir() or any(p in str(file_path) for p in exclude_patterns): | |
| continue | |
| if file_extensions and file_path.suffix not in file_extensions: | |
| continue | |
| try: | |
| docs = self.process_file(str(file_path), metadata) | |
| all_documents.extend(docs) | |
| except Exception as e: | |
| print(f"Skipping {file_path.name}: {e}") | |
| return all_documents | |