chatdoc / app /services /vector_store.py
celiolr's picture
Deploy: 51 arquivos (116.9 MB) - 2025-12-18 13:52:22
4dc3b79
Raw History Blame Contribute Delete
5.15 kB
import os
from typing import List, Optional, Dict, Any
from pathlib import Path
import chromadb
from chromadb.config import Settings
from langchain_community.vectorstores import Chroma
from langchain.schema import Document
import logging
import datetime
from app.core.config import settings
from app.core.embeddings import EmbeddingFactory
logger = logging.getLogger(__name__)
class VectorStoreManager:
def __init__(self):
persist_input = Path(settings.chroma_persist_directory).expanduser()
project_root = Path(__file__).resolve().parent.parent.parent
self.persist_directory = (project_root / persist_input if not persist_input.is_absolute() else persist_input).resolve(strict=False)
self.persist_directory.mkdir(parents=True, exist_ok=True)
self.embeddings = EmbeddingFactory.get_embeddings()
self.client = chromadb.PersistentClient(
path=str(self.persist_directory),
settings=Settings(anonymized_telemetry=False, allow_reset=True)
)
self.vector_store = Chroma(
collection_name=settings.collection_name,
embedding_function=self.embeddings,
persist_directory=str(self.persist_directory),
client=self.client
)
logger.info(f"Chroma vector store initialized at {self.persist_directory}")
def add_documents(self, documents: List[Document]) -> None:
if not documents:
return
self.vector_store.add_documents(documents)
self._update_collection_metadata()
def _update_collection_metadata(self):
try:
collection = self.client.get_collection(name=settings.collection_name)
current_metadata = collection.metadata or {}
current_metadata["last_updated_utc"] = datetime.datetime.utcnow().isoformat()
collection.modify(metadata=current_metadata)
logger.info("Collection metadata updated.")
except Exception as e:
logger.error(f"Failed to update collection metadata: {e}")
def similarity_search(self, query: str, k: int = 4, filter: Optional[Dict[str, Any]] = None) -> List[Document]:
try:
return self.vector_store.similarity_search(query=query, k=k, filter=filter)
except Exception as e:
logger.error(f"Error in similarity search: {e}")
return []
def similarity_search_with_score(self, query: str, k: int = 4, filter: Optional[Dict[str, Any]] = None) -> List:
try:
return self.vector_store.similarity_search_with_score(query=query, k=k, filter=filter)
except Exception as e:
logger.error(f"Error in similarity search with score: {e}")
return []
def get_collection_stats(self) -> Dict[str, Any]:
try:
collection = self.client.get_collection(settings.collection_name)
return {
"total_documents": collection.count(),
"collection_name": collection.name,
"metadata": collection.metadata
}
except Exception as e:
logger.error(f"Error getting collection stats: {e}")
return {}
def list_indexed_documents(self, allowed_scopes: Optional[List[str]] = None) -> List[Dict[str, str]]:
"""
List all unique indexed documents with their scopes.
Returns a list of dicts: [{'filename': '...', 'scope': '...'}]
"""
try:
collection = self.client.get_collection(settings.collection_name)
where_filter = None
if allowed_scopes:
where_filter = {"scope": {"$in": allowed_scopes}}
# Busca metadados
result = collection.get(where=where_filter, include=["metadatas"])
metadatas = result['metadatas']
if not metadatas:
return []
# Usa um dicionário para garantir unicidade por filename
unique_docs = {}
for m in metadatas:
filename = m.get('filename', 'Unknown')
scope = m.get('scope', 'default')
if filename not in unique_docs:
unique_docs[filename] = {'filename': filename, 'scope': scope}
# Retorna lista ordenada por escopo e depois por nome
sorted_docs = sorted(unique_docs.values(), key=lambda x: (x['scope'], x['filename']))
return sorted_docs
except Exception as e:
logger.error(f"Error listing indexed documents: {e}")
return []
def reset_collection(self):
try:
self.client.delete_collection(name=settings.collection_name)
self.client.create_collection(name=settings.collection_name)
logger.warning("Vector store collection reset successfully.")
return True
except Exception as e:
logger.error(f"Error resetting collection: {e}")
raise