Download app/services/vector_store.py from aionaion/chatdoc: direct link, hf CLI and curl.
- Browser
- Download file 5.15 kB
-
https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/app/services/vector_store.py
- Command line
-
hf download hf://spaces/aionaion/chatdoc/app/services/vector_store.py
-
curl -L -o vector_store.py https://huggingface.co/spaces/aionaion/chatdoc/resolve/main/app/services/vector_store.py
5.15 kB
| import os | |
| from typing import List, Optional, Dict, Any | |
| from pathlib import Path | |
| import chromadb | |
| from chromadb.config import Settings | |
| from langchain_community.vectorstores import Chroma | |
| from langchain.schema import Document | |
| import logging | |
| import datetime | |
| from app.core.config import settings | |
| from app.core.embeddings import EmbeddingFactory | |
| logger = logging.getLogger(__name__) | |
| class VectorStoreManager: | |
| def __init__(self): | |
| persist_input = Path(settings.chroma_persist_directory).expanduser() | |
| project_root = Path(__file__).resolve().parent.parent.parent | |
| self.persist_directory = (project_root / persist_input if not persist_input.is_absolute() else persist_input).resolve(strict=False) | |
| self.persist_directory.mkdir(parents=True, exist_ok=True) | |
| self.embeddings = EmbeddingFactory.get_embeddings() | |
| self.client = chromadb.PersistentClient( | |
| path=str(self.persist_directory), | |
| settings=Settings(anonymized_telemetry=False, allow_reset=True) | |
| ) | |
| self.vector_store = Chroma( | |
| collection_name=settings.collection_name, | |
| embedding_function=self.embeddings, | |
| persist_directory=str(self.persist_directory), | |
| client=self.client | |
| ) | |
| logger.info(f"Chroma vector store initialized at {self.persist_directory}") | |
| def add_documents(self, documents: List[Document]) -> None: | |
| if not documents: | |
| return | |
| self.vector_store.add_documents(documents) | |
| self._update_collection_metadata() | |
| def _update_collection_metadata(self): | |
| try: | |
| collection = self.client.get_collection(name=settings.collection_name) | |
| current_metadata = collection.metadata or {} | |
| current_metadata["last_updated_utc"] = datetime.datetime.utcnow().isoformat() | |
| collection.modify(metadata=current_metadata) | |
| logger.info("Collection metadata updated.") | |
| except Exception as e: | |
| logger.error(f"Failed to update collection metadata: {e}") | |
| def similarity_search(self, query: str, k: int = 4, filter: Optional[Dict[str, Any]] = None) -> List[Document]: | |
| try: | |
| return self.vector_store.similarity_search(query=query, k=k, filter=filter) | |
| except Exception as e: | |
| logger.error(f"Error in similarity search: {e}") | |
| return [] | |
| def similarity_search_with_score(self, query: str, k: int = 4, filter: Optional[Dict[str, Any]] = None) -> List: | |
| try: | |
| return self.vector_store.similarity_search_with_score(query=query, k=k, filter=filter) | |
| except Exception as e: | |
| logger.error(f"Error in similarity search with score: {e}") | |
| return [] | |
| def get_collection_stats(self) -> Dict[str, Any]: | |
| try: | |
| collection = self.client.get_collection(settings.collection_name) | |
| return { | |
| "total_documents": collection.count(), | |
| "collection_name": collection.name, | |
| "metadata": collection.metadata | |
| } | |
| except Exception as e: | |
| logger.error(f"Error getting collection stats: {e}") | |
| return {} | |
| def list_indexed_documents(self, allowed_scopes: Optional[List[str]] = None) -> List[Dict[str, str]]: | |
| """ | |
| List all unique indexed documents with their scopes. | |
| Returns a list of dicts: [{'filename': '...', 'scope': '...'}] | |
| """ | |
| try: | |
| collection = self.client.get_collection(settings.collection_name) | |
| where_filter = None | |
| if allowed_scopes: | |
| where_filter = {"scope": {"$in": allowed_scopes}} | |
| # Busca metadados | |
| result = collection.get(where=where_filter, include=["metadatas"]) | |
| metadatas = result['metadatas'] | |
| if not metadatas: | |
| return [] | |
| # Usa um dicionário para garantir unicidade por filename | |
| unique_docs = {} | |
| for m in metadatas: | |
| filename = m.get('filename', 'Unknown') | |
| scope = m.get('scope', 'default') | |
| if filename not in unique_docs: | |
| unique_docs[filename] = {'filename': filename, 'scope': scope} | |
| # Retorna lista ordenada por escopo e depois por nome | |
| sorted_docs = sorted(unique_docs.values(), key=lambda x: (x['scope'], x['filename'])) | |
| return sorted_docs | |
| except Exception as e: | |
| logger.error(f"Error listing indexed documents: {e}") | |
| return [] | |
| def reset_collection(self): | |
| try: | |
| self.client.delete_collection(name=settings.collection_name) | |
| self.client.create_collection(name=settings.collection_name) | |
| logger.warning("Vector store collection reset successfully.") | |
| return True | |
| except Exception as e: | |
| logger.error(f"Error resetting collection: {e}") | |
| raise | |