leaseguard-api / services /vector_service.py
HFShishir's picture
Upload 38 files
8f76784 verified
Raw History Blame Contribute Delete
3.78 kB
"""
Vector Service — Local ChromaDB
Architecture:
- Text → LangChain Documents → [RecursiveCharacterTextSplitter]
- Vector Store: ChromaDB (local persistent storage)
- Embedding: HuggingFaceEmbeddings(all-MiniLM-L6-v2)
"""
import os
from typing import List, Optional
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_chroma import Chroma
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
from config import config
# 1. Lazy Loading for Embedding Model
_embeddings_model = None
def get_embeddings():
global _embeddings_model
if _embeddings_model is None:
print("DEBUG: Initializing HuggingFaceEmbeddings (all-MiniLM-L6-v2)...")
_embeddings_model = HuggingFaceEmbeddings(
model_name="sentence-transformers/all-MiniLM-L6-v2"
)
return _embeddings_model
# 2. Text Splitter for post-parsing chunking (if needed)
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=150,
length_function=len,
)
# 3. ChromaDB persistent directory
CHROMA_PERSIST_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "chroma_db")
# Singleton instance
_vector_store = None
def _get_vector_store(collection_name: str = "document_chunks"):
"""
Initializes the Chroma store with a singleton pattern.
Uses local persistent storage.
"""
global _vector_store
if _vector_store is not None:
return _vector_store
_vector_store = Chroma(
collection_name=collection_name,
embedding_function=get_embeddings(),
persist_directory=CHROMA_PERSIST_DIR,
)
return _vector_store
def index_document(user_id: str, analysis_id: str, documents: List[Document]):
"""
Indexes documents into ChromaDB with user and analysis metadata.
"""
if not documents:
print(f"WARN: No documents to index for analysis {analysis_id}")
return
# Add metadata for isolation before indexing
for doc in documents:
doc.metadata.update({
"user_id": user_id,
"analysis_id": analysis_id
})
# Chunks documents if they are too large
split_docs = text_splitter.split_documents(documents)
print(f"DEBUG: Indexing {len(split_docs)} chunks for analysis {analysis_id} (user: {user_id})")
store = _get_vector_store()
store.add_documents(split_docs)
print(f"DEBUG: Successfully indexed chunks in ChromaDB.")
def get_retriever(user_id: str, analysis_id: str, k: int = 4):
"""
Returns a filtered LangChain Retriever for a specific analysis.
This is the core LangChain-native interface for the chat system.
"""
store = _get_vector_store()
# We use analysis_id which is a unique UUID
# Chroma requires exactly one operator if not using $and
search_kwargs = {
"k": k,
"filter": {
"analysis_id": analysis_id
}
}
return store.as_retriever(search_kwargs=search_kwargs)
def query_document_legacy(user_id: str, analysis_id: str, query: str, k: int = 4) -> List[str]:
"""
Legacy wrapper for existing codebase.
Returns raw strings instead of Document objects.
"""
retriever = get_retriever(user_id, analysis_id, k)
docs = retriever.invoke(query)
return [doc.page_content for doc in docs]
def delete_document_index(user_id: str, analysis_id: str):
"""
Removes index for a document by filtering metadata.
"""
store = _get_vector_store()
try:
store.delete(filter={"analysis_id": analysis_id})
print(f"DEBUG: Vector cleanup complete for analysis {analysis_id}")
except Exception as e:
print(f"WARN: Vector cleanup encountered an issue: {e}")