Download app/retrieval/retriever.py from acpcchatbotadmin/acpcchatbot_testing_first: direct link, hf CLI and curl.
- Browser
- Download file 3.94 kB
-
https://huggingface.co/spaces/acpcchatbotadmin/acpcchatbot_testing_first/resolve/main/app/retrieval/retriever.py
- Command line
-
hf download hf://spaces/acpcchatbotadmin/acpcchatbot_testing_first/app/retrieval/retriever.py
-
curl -L -o retriever.py https://huggingface.co/spaces/acpcchatbotadmin/acpcchatbot_testing_first/resolve/main/app/retrieval/retriever.py
3.94 kB
| import json | |
| import logging | |
| from langchain_core.documents import Document | |
| from app.retrieval.qdrant_store import get_qdrant_client, get_embeddings | |
| from app.retrieval.reranker import rerank_documents | |
| from app.config import settings | |
| logger = logging.getLogger(__name__) | |
| def retrieve_and_rerank(query: str) -> list[Document]: | |
| """ | |
| Embed query → search Qdrant with dense vectors (top_k) → | |
| deserialize payloads → Cohere rerank (top_n). | |
| Returns reranked documents with content and metadata. | |
| """ | |
| client = get_qdrant_client() | |
| embeddings = get_embeddings() | |
| # Get dense embedding for the query | |
| try: | |
| dense_embedding = embeddings.embed_query(query) | |
| except Exception as e: | |
| logger.error("Failed to embed query: %s", e) | |
| return [] | |
| # Try to search in Qdrant | |
| search_result = None | |
| # Query using the official Qdrant API with 'using' parameter for named vectors | |
| try: | |
| search_result = client.query_points( | |
| collection_name=settings.collection_name, | |
| query=dense_embedding, | |
| using=settings.vector_name, # Official Qdrant syntax for named vectors | |
| limit=settings.retrieval_top_k, | |
| with_payload=True, | |
| with_vectors=False, | |
| ) | |
| search_result = ( | |
| search_result.points if hasattr(search_result, 'points') else search_result | |
| ) | |
| logger.debug("Query executed successfully with named vector: %s", settings.vector_name) | |
| except Exception as e: | |
| logger.error("Failed to query Qdrant: %s", e) | |
| return [] | |
| if not search_result: | |
| logger.warning("No search results returned from Qdrant") | |
| return [] | |
| # Convert Qdrant results to LangChain Documents | |
| documents = [] | |
| for scored_point in search_result: | |
| payload = scored_point.payload or {} | |
| # Deserialize _node_content if present | |
| page_content = "" | |
| metadata = {} | |
| if "_node_content" in payload: | |
| try: | |
| node_data = json.loads(payload["_node_content"]) | |
| page_content = node_data.get("text", "") | |
| # Preserve original metadata fields | |
| if "metadata" in node_data: | |
| metadata = node_data["metadata"] | |
| except (json.JSONDecodeError, KeyError) as e: | |
| logger.debug("Failed to deserialize node_content: %s", e) | |
| # Add other payload fields as metadata | |
| for key, value in payload.items(): | |
| if key not in ["_node_content"] and key not in metadata: | |
| metadata[key] = value | |
| doc = Document(page_content=page_content, metadata=metadata) | |
| documents.append(doc) | |
| if not documents: | |
| logger.warning("No documents with content after deserialization") | |
| return [] | |
| logger.info("Retrieved %d documents from Qdrant", len(documents)) | |
| # Rerank documents using Cohere | |
| try: | |
| reranked = rerank_documents(query=query, documents=documents) | |
| logger.info("Reranked to %d documents", len(reranked)) | |
| return reranked | |
| except Exception as e: | |
| logger.error("Reranking failed: %s, returning original documents", e) | |
| return documents | |
| def format_sources(documents: list[Document]) -> list[dict]: | |
| """Extract source metadata from documents for the API response.""" | |
| sources = [] | |
| for i, doc in enumerate(documents): | |
| meta = doc.metadata or {} | |
| sources.append( | |
| { | |
| "rank": i + 1, | |
| "score": float(meta.get("relevance_score", meta.get("score", 0.0))), | |
| "file_name": meta.get("file_name", ""), | |
| "url": meta.get("url", meta.get("page_id", "")), | |
| "text_preview": doc.page_content[:300], | |
| } | |
| ) | |
| return sources | |