import json import logging from langchain_core.documents import Document from app.retrieval.qdrant_store import get_qdrant_client, get_embeddings from app.retrieval.reranker import rerank_documents from app.config import settings logger = logging.getLogger(__name__) def retrieve_and_rerank(query: str) -> list[Document]: """ Embed query → search Qdrant with dense vectors (top_k) → deserialize payloads → Cohere rerank (top_n). Returns reranked documents with content and metadata. """ client = get_qdrant_client() embeddings = get_embeddings() # Get dense embedding for the query try: dense_embedding = embeddings.embed_query(query) except Exception as e: logger.error("Failed to embed query: %s", e) return [] # Try to search in Qdrant search_result = None # Query using the official Qdrant API with 'using' parameter for named vectors try: search_result = client.query_points( collection_name=settings.collection_name, query=dense_embedding, using=settings.vector_name, # Official Qdrant syntax for named vectors limit=settings.retrieval_top_k, with_payload=True, with_vectors=False, ) search_result = ( search_result.points if hasattr(search_result, 'points') else search_result ) logger.debug("Query executed successfully with named vector: %s", settings.vector_name) except Exception as e: logger.error("Failed to query Qdrant: %s", e) return [] if not search_result: logger.warning("No search results returned from Qdrant") return [] # Convert Qdrant results to LangChain Documents documents = [] for scored_point in search_result: payload = scored_point.payload or {} # Deserialize _node_content if present page_content = "" metadata = {} if "_node_content" in payload: try: node_data = json.loads(payload["_node_content"]) page_content = node_data.get("text", "") # Preserve original metadata fields if "metadata" in node_data: metadata = node_data["metadata"] except (json.JSONDecodeError, KeyError) as e: logger.debug("Failed to deserialize node_content: %s", e) # Add other payload fields as metadata for key, value in payload.items(): if key not in ["_node_content"] and key not in metadata: metadata[key] = value doc = Document(page_content=page_content, metadata=metadata) documents.append(doc) if not documents: logger.warning("No documents with content after deserialization") return [] logger.info("Retrieved %d documents from Qdrant", len(documents)) # Rerank documents using Cohere try: reranked = rerank_documents(query=query, documents=documents) logger.info("Reranked to %d documents", len(reranked)) return reranked except Exception as e: logger.error("Reranking failed: %s, returning original documents", e) return documents def format_sources(documents: list[Document]) -> list[dict]: """Extract source metadata from documents for the API response.""" sources = [] for i, doc in enumerate(documents): meta = doc.metadata or {} sources.append( { "rank": i + 1, "score": float(meta.get("relevance_score", meta.get("score", 0.0))), "file_name": meta.get("file_name", ""), "url": meta.get("url", meta.get("page_id", "")), "text_preview": doc.page_content[:300], } ) return sources