import os from typing import List, Optional, Dict, Any from pathlib import Path import chromadb from chromadb.config import Settings from langchain_community.vectorstores import Chroma from langchain.schema import Document import logging import datetime from app.core.config import settings from app.core.embeddings import EmbeddingFactory logger = logging.getLogger(__name__) class VectorStoreManager: def __init__(self): persist_input = Path(settings.chroma_persist_directory).expanduser() project_root = Path(__file__).resolve().parent.parent.parent self.persist_directory = (project_root / persist_input if not persist_input.is_absolute() else persist_input).resolve(strict=False) self.persist_directory.mkdir(parents=True, exist_ok=True) self.embeddings = EmbeddingFactory.get_embeddings() self.client = chromadb.PersistentClient( path=str(self.persist_directory), settings=Settings(anonymized_telemetry=False, allow_reset=True) ) self.vector_store = Chroma( collection_name=settings.collection_name, embedding_function=self.embeddings, persist_directory=str(self.persist_directory), client=self.client ) logger.info(f"Chroma vector store initialized at {self.persist_directory}") def add_documents(self, documents: List[Document]) -> None: if not documents: return self.vector_store.add_documents(documents) self._update_collection_metadata() def _update_collection_metadata(self): try: collection = self.client.get_collection(name=settings.collection_name) current_metadata = collection.metadata or {} current_metadata["last_updated_utc"] = datetime.datetime.utcnow().isoformat() collection.modify(metadata=current_metadata) logger.info("Collection metadata updated.") except Exception as e: logger.error(f"Failed to update collection metadata: {e}") def similarity_search(self, query: str, k: int = 4, filter: Optional[Dict[str, Any]] = None) -> List[Document]: try: return self.vector_store.similarity_search(query=query, k=k, filter=filter) except Exception as e: logger.error(f"Error in similarity search: {e}") return [] def similarity_search_with_score(self, query: str, k: int = 4, filter: Optional[Dict[str, Any]] = None) -> List: try: return self.vector_store.similarity_search_with_score(query=query, k=k, filter=filter) except Exception as e: logger.error(f"Error in similarity search with score: {e}") return [] def get_collection_stats(self) -> Dict[str, Any]: try: collection = self.client.get_collection(settings.collection_name) return { "total_documents": collection.count(), "collection_name": collection.name, "metadata": collection.metadata } except Exception as e: logger.error(f"Error getting collection stats: {e}") return {} def list_indexed_documents(self, allowed_scopes: Optional[List[str]] = None) -> List[Dict[str, str]]: """ List all unique indexed documents with their scopes. Returns a list of dicts: [{'filename': '...', 'scope': '...'}] """ try: collection = self.client.get_collection(settings.collection_name) where_filter = None if allowed_scopes: where_filter = {"scope": {"$in": allowed_scopes}} # Busca metadados result = collection.get(where=where_filter, include=["metadatas"]) metadatas = result['metadatas'] if not metadatas: return [] # Usa um dicionário para garantir unicidade por filename unique_docs = {} for m in metadatas: filename = m.get('filename', 'Unknown') scope = m.get('scope', 'default') if filename not in unique_docs: unique_docs[filename] = {'filename': filename, 'scope': scope} # Retorna lista ordenada por escopo e depois por nome sorted_docs = sorted(unique_docs.values(), key=lambda x: (x['scope'], x['filename'])) return sorted_docs except Exception as e: logger.error(f"Error listing indexed documents: {e}") return [] def reset_collection(self): try: self.client.delete_collection(name=settings.collection_name) self.client.create_collection(name=settings.collection_name) logger.warning("Vector store collection reset successfully.") return True except Exception as e: logger.error(f"Error resetting collection: {e}") raise