File size: 3,939 Bytes
481dfe6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
import json
import logging
from langchain_core.documents import Document
from app.retrieval.qdrant_store import get_qdrant_client, get_embeddings
from app.retrieval.reranker import rerank_documents
from app.config import settings

logger = logging.getLogger(__name__)


def retrieve_and_rerank(query: str) -> list[Document]:
    """

    Embed query → search Qdrant with dense vectors (top_k) →

    deserialize payloads → Cohere rerank (top_n).

    Returns reranked documents with content and metadata.

    """
    client = get_qdrant_client()
    embeddings = get_embeddings()

    # Get dense embedding for the query
    try:
        dense_embedding = embeddings.embed_query(query)
    except Exception as e:
        logger.error("Failed to embed query: %s", e)
        return []

    # Try to search in Qdrant
    search_result = None

    # Query using the official Qdrant API with 'using' parameter for named vectors
    try:
        search_result = client.query_points(
            collection_name=settings.collection_name,
            query=dense_embedding,
            using=settings.vector_name,  # Official Qdrant syntax for named vectors
            limit=settings.retrieval_top_k,
            with_payload=True,
            with_vectors=False,
        )
        search_result = (
            search_result.points if hasattr(search_result, 'points') else search_result
        )
        logger.debug("Query executed successfully with named vector: %s", settings.vector_name)
    except Exception as e:
        logger.error("Failed to query Qdrant: %s", e)
        return []

    if not search_result:
        logger.warning("No search results returned from Qdrant")
        return []

    # Convert Qdrant results to LangChain Documents
    documents = []
    for scored_point in search_result:
        payload = scored_point.payload or {}

        # Deserialize _node_content if present
        page_content = ""
        metadata = {}

        if "_node_content" in payload:
            try:
                node_data = json.loads(payload["_node_content"])
                page_content = node_data.get("text", "")
                # Preserve original metadata fields
                if "metadata" in node_data:
                    metadata = node_data["metadata"]
            except (json.JSONDecodeError, KeyError) as e:
                logger.debug("Failed to deserialize node_content: %s", e)

        # Add other payload fields as metadata
        for key, value in payload.items():
            if key not in ["_node_content"] and key not in metadata:
                metadata[key] = value

        doc = Document(page_content=page_content, metadata=metadata)
        documents.append(doc)

    if not documents:
        logger.warning("No documents with content after deserialization")
        return []

    logger.info("Retrieved %d documents from Qdrant", len(documents))

    # Rerank documents using Cohere
    try:
        reranked = rerank_documents(query=query, documents=documents)
        logger.info("Reranked to %d documents", len(reranked))
        return reranked
    except Exception as e:
        logger.error("Reranking failed: %s, returning original documents", e)
        return documents



def format_sources(documents: list[Document]) -> list[dict]:
    """Extract source metadata from documents for the API response."""
    sources = []
    for i, doc in enumerate(documents):
        meta = doc.metadata or {}
        sources.append(
            {
                "rank": i + 1,
                "score": float(meta.get("relevance_score", meta.get("score", 0.0))),
                "file_name": meta.get("file_name", ""),
                "url": meta.get("url", meta.get("page_id", "")),
                "text_preview": doc.page_content[:300],
            }
        )
    return sources