Spaces:
Sleeping
Sleeping
File size: 4,372 Bytes
b3b0ccd 717b14e b3b0ccd 717b14e b3b0ccd 717b14e b3b0ccd 717b14e b3b0ccd 717b14e b3b0ccd 717b14e b3b0ccd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | import config
from db.dense_embeddings import DenseEmbeddings
from langchain_qdrant import QdrantVectorStore, FastEmbedSparse, RetrievalMode
from qdrant_client import QdrantClient
from qdrant_client.http import models as qmodels
class VectorDbManager:
__client: QdrantClient
__dense_embeddings: DenseEmbeddings
__sparse_embeddings: FastEmbedSparse
def __init__(self):
self.__client = QdrantClient(path=config.QDRANT_DB_PATH)
self.__dense_embeddings = DenseEmbeddings(model_name=config.DENSE_MODEL)
self.__sparse_embeddings = FastEmbedSparse(model_name=config.SPARSE_MODEL)
def _dense_vector_size(self):
return self.__dense_embeddings.vector_size
@staticmethod
def _collection_vector_size(collection_info):
vectors_config = collection_info.config.params.vectors
if hasattr(vectors_config, "size"):
return vectors_config.size
if isinstance(vectors_config, dict) and vectors_config:
first_vector = next(iter(vectors_config.values()))
return getattr(first_vector, "size", None)
return None
def create_collection(self, collection_name):
expected_size = self._dense_vector_size()
if not self.__client.collection_exists(collection_name):
print(f"Creating collection: {collection_name}...")
self.__client.create_collection(
collection_name=collection_name,
vectors_config=qmodels.VectorParams(size=expected_size, distance=qmodels.Distance.COSINE),
sparse_vectors_config={config.SPARSE_VECTOR_NAME: qmodels.SparseVectorParams()},
)
print(f"✓ Collection created: {collection_name}")
else:
collection_info = self.__client.get_collection(collection_name)
vectors_config = collection_info.config.params.vectors
dense_config = (
vectors_config.get("") if isinstance(vectors_config, dict)
else vectors_config
)
if dense_config is None:
raise ValueError(f"Qdrant collection '{collection_name}' must have an unnamed dense vector.")
if dense_config.distance != qmodels.Distance.COSINE:
raise ValueError(f"Qdrant collection '{collection_name}' must use cosine distance.")
sparse_config = collection_info.config.params.sparse_vectors or {}
if config.SPARSE_VECTOR_NAME not in sparse_config:
raise ValueError(
f"Qdrant collection '{collection_name}' is missing sparse vector '{config.SPARSE_VECTOR_NAME}'."
)
existing_size = self._collection_vector_size(collection_info)
if existing_size and existing_size != expected_size:
raise ValueError(
f"Qdrant collection '{collection_name}' has dense vector size "
f"{existing_size}, but '{config.DENSE_MODEL}' produces size "
f"{expected_size}. Clear and re-index the collection after "
"changing embedding models."
)
print(f"✓ Collection already exists: {collection_name}")
def delete_collection(self, collection_name):
try:
if self.__client.collection_exists(collection_name):
print(f"Removing existing Qdrant collection: {collection_name}")
self.__client.delete_collection(collection_name)
except Exception as e:
raise RuntimeError(f"Unable to delete Qdrant collection '{collection_name}'.") from e
def get_collection(self, collection_name) -> QdrantVectorStore:
try:
return QdrantVectorStore(
client=self.__client,
collection_name=collection_name,
embedding=self.__dense_embeddings,
sparse_embedding=self.__sparse_embeddings,
retrieval_mode=RetrievalMode.HYBRID,
sparse_vector_name=config.SPARSE_VECTOR_NAME,
# Schema is checked above without requesting GPU inference at startup.
validate_collection_config=False,
)
except Exception as e:
raise RuntimeError(f"Unable to initialize Qdrant collection '{collection_name}'.") from e
|