File size: 8,470 Bytes
1ac24a4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
"""
Vector Store Module

Manages the ChromaDB vector database for document storage and retrieval.
Provides a unified interface for:
- Storing document embeddings
- Similarity search
- Collection management
"""

import logging
from pathlib import Path
from typing import List, Optional, Dict, Any

from langchain_chroma import Chroma
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings

from config.settings import settings

logger = logging.getLogger(__name__)


class VectorStoreError(Exception):
    """Custom exception for vector store operations."""
    pass


class VectorStore:
    """
    ChromaDB-based vector store for document embeddings.
    
    Provides persistent storage and similarity search capabilities
    for RAG applications.
    
    Example:
        >>> from src.rag.embeddings import get_embeddings
        >>> embeddings = get_embeddings()
        >>> store = VectorStore(embeddings, collection_name="my_docs")
        >>> store.add_documents(chunks)
        >>> results = store.similarity_search("What is RAG?", k=5)
    """
    
    def __init__(
        self,
        embedding_function: Embeddings,
        collection_name: str = "documents",
        persist_directory: Optional[str] = None,
    ):
        """
        Initialize the vector store.
        
        Args:
            embedding_function: LangChain embeddings instance.
            collection_name: Name of the ChromaDB collection.
            persist_directory: Directory to persist the database.
        """
        self.embedding_function = embedding_function
        self.collection_name = collection_name
        
        # Set persist directory from settings or parameter
        if persist_directory:
            self.persist_directory = Path(persist_directory)
        else:
            self.persist_directory = settings.chroma_dir
        
        # Ensure directory exists
        self.persist_directory.mkdir(parents=True, exist_ok=True)
        
        # Initialize ChromaDB
        self._store = self._create_store()
        
        logger.info(
            f"Initialized VectorStore: collection='{collection_name}', "
            f"persist_dir='{self.persist_directory}'"
        )
    
    def _create_store(self) -> Chroma:
        """Create or load the ChromaDB store."""
        return Chroma(
            collection_name=self.collection_name,
            embedding_function=self.embedding_function,
            persist_directory=str(self.persist_directory),
        )
    
    def add_documents(
        self,
        documents: List[Document],
        batch_size: int = 100
    ) -> List[str]:
        """
        Add documents to the vector store.
        
        Args:
            documents: List of documents to add.
            batch_size: Number of documents to add per batch.
            
        Returns:
            List of document IDs.
        """
        if not documents:
            logger.warning("No documents provided to add")
            return []
        
        all_ids: List[str] = []
        
        # Process in batches to avoid memory issues
        for i in range(0, len(documents), batch_size):
            batch = documents[i:i + batch_size]
            
            try:
                ids = self._store.add_documents(batch)
                all_ids.extend(ids)
                logger.debug(f"Added batch {i // batch_size + 1}: {len(batch)} documents")
            except Exception as e:
                logger.error(f"Error adding batch {i // batch_size + 1}: {e}")
                raise VectorStoreError(f"Failed to add documents: {e}") from e
        
        logger.info(f"Added {len(all_ids)} documents to collection '{self.collection_name}'")
        return all_ids
    
    def similarity_search(
        self,
        query: str,
        k: int = None,
        filter: Optional[Dict[str, Any]] = None
    ) -> List[Document]:
        """
        Search for similar documents.
        
        Args:
            query: Search query text.
            k: Number of results to return (default from settings).
            filter: Optional metadata filter.
            
        Returns:
            List of similar documents.
        """
        k = k or settings.top_k_results
        
        try:
            results = self._store.similarity_search(
                query=query,
                k=k,
                filter=filter
            )
            
            logger.debug(f"Found {len(results)} results for query: '{query[:50]}...'")
            return results
            
        except Exception as e:
            logger.error(f"Error during similarity search: {e}")
            raise VectorStoreError(f"Search failed: {e}") from e
    
    def similarity_search_with_score(
        self,
        query: str,
        k: int = None,
        filter: Optional[Dict[str, Any]] = None
    ) -> List[tuple]:
        """
        Search for similar documents with relevance scores.
        
        Args:
            query: Search query text.
            k: Number of results to return.
            filter: Optional metadata filter.
            
        Returns:
            List of (Document, score) tuples.
        """
        k = k or settings.top_k_results
        
        try:
            results = self._store.similarity_search_with_score(
                query=query,
                k=k,
                filter=filter
            )
            
            logger.debug(
                f"Found {len(results)} results with scores for query: '{query[:50]}...'"
            )
            return results
            
        except Exception as e:
            logger.error(f"Error during similarity search with score: {e}")
            raise VectorStoreError(f"Search failed: {e}") from e
    
    def delete_collection(self) -> None:
        """Delete the entire collection."""
        try:
            self._store.delete_collection()
            logger.info(f"Deleted collection '{self.collection_name}'")
            
            # Recreate empty store
            self._store = self._create_store()
            
        except Exception as e:
            logger.error(f"Error deleting collection: {e}")
            raise VectorStoreError(f"Failed to delete collection: {e}") from e
    
    def get_collection_stats(self) -> Dict[str, Any]:
        """
        Get statistics about the collection.
        
        Returns:
            Dictionary with collection statistics.
        """
        try:
            # Access the underlying ChromaDB collection
            collection = self._store._collection
            
            return {
                "collection_name": self.collection_name,
                "document_count": collection.count(),
                "persist_directory": str(self.persist_directory),
            }
            
        except Exception as e:
            logger.warning(f"Error getting collection stats: {e}")
            return {
                "collection_name": self.collection_name,
                "document_count": "unknown",
                "persist_directory": str(self.persist_directory),
            }
    
    @property
    def retriever(self):
        """
        Get a LangChain retriever from this store.
        
        Returns:
            VectorStoreRetriever instance.
        """
        return self._store.as_retriever(
            search_type="similarity",
            search_kwargs={"k": settings.top_k_results}
        )
    
    def as_retriever(self, **kwargs):
        """
        Get a customized retriever.
        
        Args:
            **kwargs: Arguments passed to as_retriever().
            
        Returns:
            VectorStoreRetriever instance.
        """
        return self._store.as_retriever(**kwargs)


# Factory function for creating vector stores
def create_vector_store(
    embedding_function: Embeddings,
    collection_name: str = "documents",
    persist_directory: Optional[str] = None,
) -> VectorStore:
    """
    Factory function to create a vector store.
    
    Args:
        embedding_function: LangChain embeddings instance.
        collection_name: Name of the collection.
        persist_directory: Directory to persist data.
        
    Returns:
        Configured VectorStore instance.
    """
    return VectorStore(
        embedding_function=embedding_function,
        collection_name=collection_name,
        persist_directory=persist_directory,
    )