Spaces:
Runtime error
Runtime error
File size: 5,906 Bytes
73ddeba 03c219f 73ddeba 03c219f 73ddeba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | import os
import uuid
import logging
from typing import List, Dict, Any, Optional
from qdrant_client import QdrantClient
from qdrant_client.http import models
from qdrant_client.http.exceptions import UnexpectedResponse
logger = logging.getLogger("rag_backend")
class QdrantVectorStore:
def __init__(self):
self.url = os.getenv("QDRANT_URL")
self.api_key = os.getenv("QDRANT_API_KEY")
self.collection_name = os.getenv("QDRANT_COLLECTION", "rag_documents")
self.vector_size = 384 # Default dimension for all-MiniLM-L6-v2
# Connect to Qdrant Cloud if credentials are provided, otherwise fallback to in-memory
if self.url and self.api_key:
url = self.url
# Qdrant Cloud URLs typically require port 443 for HTTPS REST APIs.
# If the user provided a cloud URL without specifying any port, default to 443.
clean_url = url.replace("https://", "").replace("http://", "")
if "qdrant.io" in url.lower() and ":" not in clean_url:
url = f"{url.rstrip('/')}:443"
logger.info(f"Detected Qdrant Cloud URL without port. Appending secure port 443: {url}")
else:
logger.info(f"Connecting to Qdrant Cloud at {url}...")
self.client = QdrantClient(
url=url,
api_key=self.api_key
)
self.is_cloud = True
else:
logger.warning("QDRANT_URL and QDRANT_API_KEY not found in environment variables. Falling back to local In-Memory Qdrant...")
self.client = QdrantClient(location=":memory:")
self.is_cloud = False
self._ensure_collection()
def _ensure_collection(self):
"""Ensures that the target collection exists in Qdrant."""
try:
# Check if collection exists by getting its info
self.client.get_collection(self.collection_name)
logger.info(f"Collection '{self.collection_name}' already exists.")
except (UnexpectedResponse, Exception):
# Collection does not exist or error, create it
logger.info(f"Collection '{self.collection_name}' not found. Creating a new one...")
self.client.create_collection(
collection_name=self.collection_name,
vectors_config=models.VectorParams(
size=self.vector_size,
distance=models.Distance.COSINE
)
)
logger.info(f"Collection '{self.collection_name}' created successfully.")
def add_documents(self, texts: List[str], embeddings: List[List[float]], metadatas: Optional[List[Dict[str, Any]]] = None) -> int:
"""
Upserts document chunks and their corresponding embeddings and payloads into Qdrant.
"""
if not texts:
return 0
if metadatas is None:
metadatas = [{} for _ in range(len(texts))]
points = []
for i, (text, embedding, metadata) in enumerate(zip(texts, embeddings, metadatas)):
point_id = str(uuid.uuid4())
payload = {
"text": text,
"metadata": metadata
}
points.append(
models.PointStruct(
id=point_id,
vector=embedding,
payload=payload
)
)
# Qdrant client upsert in batches
self.client.upsert(
collection_name=self.collection_name,
wait=True,
points=points
)
return len(points)
def search(self, query_embedding: List[float], top_k: int = 3) -> List[Dict[str, Any]]:
"""
Searches the Qdrant collection for similar embeddings.
Returns a list of dicts with text, metadata, and score.
"""
results = self.client.query_points(
collection_name=self.collection_name,
query=query_embedding,
limit=top_k
)
search_results = []
for hit in results.points:
search_results.append({
"text": hit.payload.get("text", ""),
"metadata": hit.payload.get("metadata", {}),
"score": hit.score
})
return search_results
def clear(self) -> bool:
"""
Deletes and recreates the collection to reset the database.
"""
try:
self.client.delete_collection(self.collection_name)
self.client.create_collection(
collection_name=self.collection_name,
vectors_config=models.VectorParams(
size=self.vector_size,
distance=models.Distance.COSINE
)
)
logger.info(f"Collection '{self.collection_name}' has been reset.")
return True
except Exception as e:
logger.error(f"Error resetting collection '{self.collection_name}': {e}")
return False
def get_status(self) -> Dict[str, Any]:
"""
Returns connection and collection stats.
"""
try:
collection_info = self.client.get_collection(self.collection_name)
vectors_count = collection_info.points_count
status = collection_info.status.value if hasattr(collection_info.status, "value") else str(collection_info.status)
except Exception as e:
logger.error(f"Error retrieving collection info: {e}")
vectors_count = 0
status = "unknown"
return {
"connection_type": "Cloud" if self.is_cloud else "In-Memory (Local)",
"collection_name": self.collection_name,
"vector_size": self.vector_size,
"vectors_count": vectors_count,
"status": status
}
|