RAG-Documind-AI / src /rag /loaders.py
Shubham Pawar
Claude Sonnet 5
fix: make ingestion and querying robust; add acceptance-test corpus
c8f7f2e
Raw History Blame Contribute Delete
7.36 kB
"""
Document Loaders Module
Handles loading documents from various formats:
- PDF files
- Markdown files
- Text files
- Directories containing mixed formats
"""
import logging
from pathlib import Path
from typing import List, Optional, Union
from langchain_community.document_loaders import (
PyPDFLoader,
TextLoader,
)
from langchain_core.documents import Document
logger = logging.getLogger(__name__)
class DocumentLoaderError(Exception):
"""Custom exception for document loading errors."""
pass
class MultiFormatDocumentLoader:
"""
A unified document loader that handles multiple file formats.
Supports:
- PDF files (.pdf)
- Markdown files (.md, .markdown)
- Text files (.txt)
- Directories containing mixed formats
Example:
>>> loader = MultiFormatDocumentLoader()
>>> docs = loader.load_file("document.pdf")
>>> docs = loader.load_directory("./documents")
"""
# Supported file extensions and their loaders.
#
# Markdown is handled with the plain TextLoader rather than
# UnstructuredMarkdownLoader: the latter pulls in `unstructured`, which
# downloads an NLTK/spaCy model at runtime on first use and fails hard in
# offline or locked-down environments (and behind strict TLS). Markdown is
# already text, so retrieval quality is unaffected.
SUPPORTED_EXTENSIONS = {
".pdf": PyPDFLoader,
".md": TextLoader,
".markdown": TextLoader,
".txt": TextLoader,
}
def __init__(self, encoding: str = "utf-8"):
"""
Initialize the document loader.
Args:
encoding: Default encoding for text files.
"""
self.encoding = encoding
def load_file(self, file_path: Union[str, Path]) -> List[Document]:
"""
Load a single file and return a list of documents.
Args:
file_path: Path to the file to load.
Returns:
List of Document objects.
Raises:
DocumentLoaderError: If file cannot be loaded.
ValueError: If file format is not supported.
"""
file_path = Path(file_path)
if not file_path.exists():
raise DocumentLoaderError(f"File not found: {file_path}")
if not file_path.is_file():
raise DocumentLoaderError(f"Not a file: {file_path}")
extension = file_path.suffix.lower()
if extension not in self.SUPPORTED_EXTENSIONS:
raise ValueError(
f"Unsupported file format: {extension}. "
f"Supported formats: {list(self.SUPPORTED_EXTENSIONS.keys())}"
)
loader_class = self.SUPPORTED_EXTENSIONS[extension]
try:
# TextLoader needs an explicit encoding parameter
if loader_class is TextLoader:
loader = loader_class(str(file_path), encoding=self.encoding)
else:
loader = loader_class(str(file_path))
documents = loader.load()
# Add source metadata
for doc in documents:
doc.metadata["source"] = str(file_path)
doc.metadata["file_name"] = file_path.name
doc.metadata["file_type"] = extension
logger.info(f"Loaded {len(documents)} document(s) from {file_path}")
return documents
except Exception as e:
logger.error(f"Error loading file {file_path}: {e}")
raise DocumentLoaderError(f"Failed to load {file_path}: {e}") from e
def load_directory(
self,
directory_path: Union[str, Path],
recursive: bool = True,
extensions: Optional[List[str]] = None
) -> List[Document]:
"""
Load all supported documents from a directory.
Args:
directory_path: Path to the directory.
recursive: Whether to search subdirectories.
extensions: Specific extensions to load (default: all supported).
Returns:
List of Document objects from all files.
Raises:
DocumentLoaderError: If directory cannot be accessed.
"""
directory_path = Path(directory_path)
if not directory_path.exists():
raise DocumentLoaderError(f"Directory not found: {directory_path}")
if not directory_path.is_dir():
raise DocumentLoaderError(f"Not a directory: {directory_path}")
# Determine which extensions to load
if extensions is None:
extensions = list(self.SUPPORTED_EXTENSIONS.keys())
else:
# Validate extensions
for ext in extensions:
if ext not in self.SUPPORTED_EXTENSIONS:
raise ValueError(f"Unsupported extension: {ext}")
all_documents: List[Document] = []
# Find all matching files
pattern = "**/*" if recursive else "*"
for ext in extensions:
glob_pattern = f"{pattern}{ext}"
files = list(directory_path.glob(glob_pattern))
for file_path in files:
try:
docs = self.load_file(file_path)
all_documents.extend(docs)
except Exception as e:
logger.warning(f"Skipping file {file_path}: {e}")
continue
logger.info(
f"Loaded {len(all_documents)} document(s) from {directory_path}"
)
return all_documents
def load_files(
self,
file_paths: List[Union[str, Path]]
) -> List[Document]:
"""
Load multiple files.
Args:
file_paths: List of file paths to load.
Returns:
List of Document objects from all files.
"""
all_documents: List[Document] = []
for file_path in file_paths:
try:
docs = self.load_file(file_path)
all_documents.extend(docs)
except Exception as e:
logger.warning(f"Skipping file {file_path}: {e}")
continue
return all_documents
@staticmethod
def get_supported_extensions() -> List[str]:
"""Return list of supported file extensions."""
return list(MultiFormatDocumentLoader.SUPPORTED_EXTENSIONS.keys())
# Convenience function for quick loading
def load_documents(
path: Union[str, Path],
recursive: bool = True
) -> List[Document]:
"""
Convenience function to load documents from a file or directory.
Args:
path: Path to file or directory.
recursive: Whether to search subdirectories (for directories).
Returns:
List of Document objects.
"""
loader = MultiFormatDocumentLoader()
path = Path(path)
if path.is_file():
return loader.load_file(path)
elif path.is_dir():
return loader.load_directory(path, recursive=recursive)
else:
raise DocumentLoaderError(f"Path does not exist: {path}")