Spaces:
Running on Zero
Running on Zero
Download retrieve.py from Varun2007/document-qa-system: direct link, hf CLI and curl.
- Browser
- Download file 18.4 kB
-
https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/retrieve.py
- Command line
-
hf download hf://spaces/Varun2007/document-qa-system/retrieve.py
-
curl -L -o retrieve.py https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/retrieve.py
18.4 kB
| """Hybrid Retrieval and Query Processing Module. | |
| Features: | |
| - Dense Vector Retrieval via FAISS (IndexFlatIP with normalized cosine embeddings) | |
| - Sparse Lexical Retrieval via BM25 (BM25Okapi) | |
| - Hybrid Fusion using Reciprocal Rank Fusion (RRF) | |
| - Cross-Encoder Reranking (ms-marco-MiniLM-L-6-v2) | |
| - Query Rewriting for conversational follow-ups | |
| - Multi-query decomposition for complex questions | |
| """ | |
| import json | |
| import math | |
| import os | |
| import re | |
| from typing import List, Dict, Any, Optional, Tuple, Set | |
| import numpy as np | |
| import faiss | |
| from rank_bm25 import BM25Okapi | |
| from sentence_transformers import SentenceTransformer, CrossEncoder | |
| from ingest import DocumentChunk | |
| def tokenize_for_bm25(text: str) -> List[str]: | |
| """Lowercase and extract alphanumeric tokens for BM25.""" | |
| return re.findall(r"\b[a-zA-Z0-9_]+\b", text.lower()) | |
| class FAISSDenseIndex: | |
| """FAISS-based dense vector index with cosine similarity.""" | |
| def __init__(self, model_name: str = "sentence-transformers/all-MiniLM-L6-v2"): | |
| self.model_name = model_name | |
| self.embedder = SentenceTransformer(model_name) | |
| self.dimension = ( | |
| self.embedder.get_embedding_dimension() | |
| if hasattr(self.embedder, "get_embedding_dimension") | |
| else self.embedder.get_sentence_embedding_dimension() | |
| ) | |
| self.index = faiss.IndexFlatIP(self.dimension) | |
| self.chunks: List[DocumentChunk] = [] | |
| def build(self, chunks: List[DocumentChunk], batch_size: int = 64) -> None: | |
| """Encode chunks and populate the FAISS index.""" | |
| self.chunks = list(chunks) | |
| if not chunks: | |
| self.index = faiss.IndexFlatIP(self.dimension) | |
| return | |
| texts = [c.text for c in chunks] | |
| embeddings = self.embedder.encode( | |
| texts, | |
| batch_size=batch_size, | |
| show_progress_bar=False, | |
| normalize_embeddings=True | |
| ) | |
| embeddings = np.array(embeddings, dtype=np.float32) | |
| self.index = faiss.IndexFlatIP(self.dimension) | |
| self.index.add(embeddings) | |
| def search(self, query: str, top_k: int = 20) -> List[Tuple[DocumentChunk, float]]: | |
| """Dense search returning (chunk, cosine_score) ranked highest to lowest.""" | |
| if not self.chunks or self.index.ntotal == 0: | |
| return [] | |
| query_vec = self.embedder.encode([query], normalize_embeddings=True) | |
| query_vec = np.array(query_vec, dtype=np.float32) | |
| k = min(top_k, len(self.chunks)) | |
| scores, indices = self.index.search(query_vec, k) | |
| results = [] | |
| for idx, score in zip(indices[0], scores[0]): | |
| if idx >= 0 and idx < len(self.chunks): | |
| results.append((self.chunks[idx], float(score))) | |
| return results | |
| def save(self, directory: str) -> None: | |
| os.makedirs(directory, exist_ok=True) | |
| faiss_path = os.path.join(directory, "faiss_index.bin") | |
| meta_path = os.path.join(directory, "chunks.json") | |
| faiss.write_index(self.index, faiss_path) | |
| with open(meta_path, "w", encoding="utf-8") as f: | |
| json.dump([c.to_dict() for c in self.chunks], f, indent=2) | |
| def load(self, directory: str) -> None: | |
| faiss_path = os.path.join(directory, "faiss_index.bin") | |
| meta_path = os.path.join(directory, "chunks.json") | |
| if os.path.exists(faiss_path) and os.path.exists(meta_path): | |
| self.index = faiss.read_index(faiss_path) | |
| with open(meta_path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| self.chunks = [DocumentChunk.from_dict(d) for d in data] | |
| class BM25SparseIndex: | |
| """BM25 lexical index for keyword precision and exact match.""" | |
| def __init__(self): | |
| self.chunks: List[DocumentChunk] = [] | |
| self.bm25: Optional[BM25Okapi] = None | |
| self.corpus_tokens: List[List[str]] = [] | |
| def build(self, chunks: List[DocumentChunk]) -> None: | |
| self.chunks = list(chunks) | |
| if not chunks: | |
| self.bm25 = None | |
| self.corpus_tokens = [] | |
| return | |
| self.corpus_tokens = [tokenize_for_bm25(c.text) for c in chunks] | |
| self.bm25 = BM25Okapi(self.corpus_tokens) | |
| def search(self, query: str, top_k: int = 20) -> List[Tuple[DocumentChunk, float]]: | |
| """Sparse BM25 search returning (chunk, bm25_score).""" | |
| if not self.chunks or self.bm25 is None: | |
| return [] | |
| tokens = tokenize_for_bm25(query) | |
| if not tokens: | |
| return [] | |
| scores = self.bm25.get_scores(tokens) | |
| top_indices = np.argsort(scores)[::-1][: min(top_k, len(self.chunks))] | |
| results = [] | |
| for idx in top_indices: | |
| score = float(scores[idx]) | |
| if score > 0.0: | |
| results.append((self.chunks[idx], score)) | |
| return results | |
| def extract_text_from_content(content: Any) -> str: | |
| """Safely extracts a flat string from string, list, or dict content representations.""" | |
| if isinstance(content, str): | |
| return content.strip() | |
| if isinstance(content, list): | |
| parts = [] | |
| for item in content: | |
| if isinstance(item, str): | |
| parts.append(item) | |
| elif isinstance(item, dict): | |
| parts.append(str(item.get("text", item.get("content", "")))) | |
| else: | |
| parts.append(str(item)) | |
| return " ".join(p for p in parts if p).strip() | |
| if isinstance(content, dict): | |
| return str(content.get("text", content.get("content", ""))).strip() | |
| return str(content or "").strip() | |
| class QueryProcessor: | |
| """Handles conversational query rewriting and multi-step question decomposition.""" | |
| def __init__(self, llm_client=None): | |
| self.llm_client = llm_client | |
| def rewrite_query(self, query: str, conversation_history: List[Dict[str, Any]]) -> str: | |
| """ | |
| Rewrites conversational follow-up questions into standalone queries. | |
| E.g.: "What about the second one?" -> "What are the details of the second recommendation in Section 3?" | |
| """ | |
| if not conversation_history: | |
| return query.strip() | |
| # Check for coreference signals (pronouns, elliptical follow-ups) | |
| coref_pattern = ( | |
| r"\b(it|its|they|them|their)\b|" | |
| r"\b(?:this|that|these|those)\s+(?:one|concept|mechanism|feature|section|finding|recommendation|policy|report|point|architecture|system|service|component|table)\b|" | |
| r"\bthe\s+(?:former|latter|first|second|third|last|previous|above)\b|" | |
| r"^(?:why|why so|how so|what about|tell me more|more details|explain further|elaborate)(?:\?|$|\s)|" | |
| r"^(?:and\s+what|what\s+else)\b" | |
| ) | |
| needs_rewrite = bool(re.search(coref_pattern, query.strip(), re.IGNORECASE)) | |
| if not needs_rewrite: | |
| return query.strip() | |
| # If LLM is available, use LLM for natural contextual rewriting | |
| if self.llm_client and hasattr(self.llm_client, "complete_chat"): | |
| prompt = ( | |
| "Given the following conversation history and a follow-up question, rewrite the follow-up " | |
| "question into a standalone search query that preserves all relevant context. Do NOT answer " | |
| "the question, only output the rewritten search query.\n\n" | |
| "Conversation History:\n" | |
| ) | |
| for turn in conversation_history[-3:]: | |
| role = "User" if turn.get("role") == "user" else "Assistant" | |
| turn_text = extract_text_from_content(turn.get("content", "")) | |
| prompt += f"{role}: {turn_text}\n" | |
| prompt += f"\nFollow-up question: {query}\nStandalone search query:" | |
| try: | |
| rewritten = self.llm_client.complete_chat([ | |
| {"role": "system", "content": "You are an expert search query reformulator."}, | |
| {"role": "user", "content": prompt} | |
| ]) | |
| if rewritten and len(rewritten.strip()) > 3: | |
| return rewritten.strip().strip('"').strip("'") | |
| except Exception: | |
| pass | |
| # Deterministic coreference resolution fallback | |
| last_user_turn = "" | |
| last_bot_turn = "" | |
| for turn in reversed(conversation_history): | |
| content = extract_text_from_content(turn.get("content", "")) | |
| if not content: | |
| continue | |
| if turn.get("role") == "user" and not last_user_turn: | |
| last_user_turn = content | |
| elif turn.get("role") == "assistant" and not last_bot_turn: | |
| last_bot_turn = content | |
| if last_user_turn and last_bot_turn: | |
| break | |
| if not last_user_turn: | |
| return query.strip() | |
| # Extract primary nouns / subjects from previous user turn, excluding common stop words | |
| stop_words = { | |
| "what", "when", "where", "which", "who", "whom", "whose", "why", "how", | |
| "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", | |
| "do", "does", "did", "the", "a", "an", "and", "or", "in", "on", "at", | |
| "to", "for", "of", "with", "by", "from", "up", "about", "into", "over", | |
| "after", "tell", "explain", "describe", "find", "show", "can", "could", | |
| "would", "should", "please" | |
| } | |
| tokens = re.findall(r"\b[A-Za-z0-9_]{3,}\b", last_user_turn) | |
| prev_keywords = [t for t in tokens if t.lower() not in stop_words] | |
| prev_context = " ".join(prev_keywords[:4]) if prev_keywords else last_user_turn[:50] | |
| return f"{query.strip()} regarding {prev_context}".strip() | |
| def decompose_query(self, query: str) -> List[str]: | |
| """ | |
| Decomposes complex multi-part questions into sub-queries. | |
| E.g.: "Compare the Q3 revenue of Division A and Division B and explain the drivers" | |
| -> ["Q3 revenue of Division A", "Q3 revenue of Division B", "revenue drivers"] | |
| """ | |
| query = query.strip() | |
| # Check if question contains conjunctions or comparative indicators | |
| split_patterns = [ | |
| r"\b(?:and also|as well as|compared to|in comparison with|versus|vs\.?)\b", | |
| r"(?<=[?.;])\s+(?=[A-Z])", | |
| r"\b(?:compare|differentiate between)\b\s+([^,]+)\s+and\s+([^?.]+)", | |
| ] | |
| sub_queries: List[str] = [] | |
| # Check comparison pattern: "Compare X and Y" | |
| cmp_match = re.search(r"\bcompare\s+(.+?)\s+(?:and|with|to)\s+(.+)", query, re.IGNORECASE) | |
| if cmp_match: | |
| part1 = cmp_match.group(1).strip() | |
| part2 = cmp_match.group(2).strip().rstrip("?.") | |
| sub_queries.append(f"{part1}") | |
| sub_queries.append(f"{part2}") | |
| sub_queries.append(query) | |
| return sub_queries | |
| # Check multi-question pattern: "What is X? How does Y work?" | |
| if "?" in query: | |
| parts = [p.strip() + "?" for p in query.split("?") if p.strip()] | |
| if len(parts) > 1: | |
| return parts | |
| # Check composite clause pattern: "A and B" | |
| if " and " in query.lower() and len(query.split()) > 7: | |
| splits = re.split(r"\s+and\s+", query, flags=re.IGNORECASE) | |
| if len(splits) == 2 and all(len(s.split()) >= 3 for s in splits): | |
| return [s.strip() for s in splits] + [query] | |
| return [query] | |
| class HybridRetriever: | |
| """ | |
| Production Hybrid Retriever: | |
| 1. Dense Vector Search (FAISS, normalized cosine) | |
| 2. Sparse Lexical Search (BM25) | |
| 3. Reciprocal Rank Fusion (RRF) -> top 20 candidates | |
| 4. Cross-Encoder Reranker (ms-marco-MiniLM-L-6-v2) -> top 5 chunks | |
| """ | |
| def __init__( | |
| self, | |
| embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2", | |
| cross_encoder_model: str = "cross-encoder/ms-marco-MiniLM-L-6-v2", | |
| rrf_k: int = 60, | |
| dense_weight: float = 0.5, | |
| sparse_weight: float = 0.5, | |
| ): | |
| self.rrf_k = rrf_k | |
| self.dense_weight = dense_weight | |
| self.sparse_weight = sparse_weight | |
| # Load models | |
| self.dense_index = FAISSDenseIndex(model_name=embedding_model) | |
| self.sparse_index = BM25SparseIndex() | |
| self.reranker = CrossEncoder(cross_encoder_model) | |
| self.query_processor = QueryProcessor() | |
| self.chunks: List[DocumentChunk] = [] | |
| def index_documents(self, documents_or_chunks: List[Any]) -> None: | |
| """Indexes parsed DocumentChunks into both dense and sparse indices.""" | |
| all_chunks: List[DocumentChunk] = [] | |
| for item in documents_or_chunks: | |
| if hasattr(item, "chunks"): | |
| all_chunks.extend(item.chunks) | |
| elif isinstance(item, DocumentChunk): | |
| all_chunks.append(item) | |
| elif isinstance(item, dict): | |
| all_chunks.append(DocumentChunk.from_dict(item)) | |
| self.chunks = all_chunks | |
| self.dense_index.build(all_chunks) | |
| self.sparse_index.build(all_chunks) | |
| def retrieve( | |
| self, | |
| query: str, | |
| initial_top_k: int = 20, | |
| rerank_top_k: int = 5, | |
| mode: str = "hybrid_rerank", | |
| conversation_history: Optional[List[Dict[str, str]]] = None, | |
| decompose: bool = True, | |
| ) -> Dict[str, Any]: | |
| """ | |
| Executes query retrieval according to the specified mode. | |
| Modes: | |
| - 'dense_only': Plain FAISS dense retrieval | |
| - 'bm25_only': Plain BM25 lexical retrieval | |
| - 'hybrid_no_rerank': Dense + BM25 RRF fusion, top-k | |
| - 'hybrid_rerank': Dense + BM25 RRF (top 20) -> Cross-Encoder rerank (top 5) | |
| """ | |
| # 1. Query Rewriting if conversational history present | |
| rewritten_query = query | |
| if conversation_history: | |
| rewritten_query = self.query_processor.rewrite_query(query, conversation_history) | |
| # 2. Query Decomposition for complex questions | |
| sub_queries = [rewritten_query] | |
| if decompose and mode in ["hybrid_rerank", "hybrid_no_rerank"]: | |
| sub_queries = self.query_processor.decompose_query(rewritten_query) | |
| if len(self.chunks) == 0: | |
| return { | |
| "query": query, | |
| "rewritten_query": rewritten_query, | |
| "sub_queries": sub_queries, | |
| "chunks": [], | |
| "scores": [], | |
| "mode": mode, | |
| } | |
| # Multi-query aggregation map: chunk_id -> DocumentChunk | |
| candidate_map: Dict[str, DocumentChunk] = {} | |
| candidate_scores: Dict[str, float] = {} | |
| for sq in sub_queries: | |
| if mode == "dense_only": | |
| res = self.dense_index.search(sq, top_k=rerank_top_k) | |
| for c, s in res: | |
| candidate_map[c.chunk_id] = c | |
| candidate_scores[c.chunk_id] = max(candidate_scores.get(c.chunk_id, 0.0), s) | |
| elif mode == "bm25_only": | |
| res = self.sparse_index.search(sq, top_k=rerank_top_k) | |
| for c, s in res: | |
| candidate_map[c.chunk_id] = c | |
| candidate_scores[c.chunk_id] = max(candidate_scores.get(c.chunk_id, 0.0), s) | |
| else: | |
| # Hybrid Search: Top 20 dense + Top 20 BM25 | |
| dense_res = self.dense_index.search(sq, top_k=initial_top_k) | |
| bm25_res = self.sparse_index.search(sq, top_k=initial_top_k) | |
| # Reciprocal Rank Fusion (RRF) | |
| rrf_scores: Dict[str, float] = {} | |
| chunk_lut: Dict[str, DocumentChunk] = {} | |
| for rank, (chunk, _) in enumerate(dense_res): | |
| chunk_lut[chunk.chunk_id] = chunk | |
| rrf_scores[chunk.chunk_id] = rrf_scores.get(chunk.chunk_id, 0.0) + ( | |
| self.dense_weight / (self.rrf_k + rank + 1) | |
| ) | |
| for rank, (chunk, _) in enumerate(bm25_res): | |
| chunk_lut[chunk.chunk_id] = chunk | |
| rrf_scores[chunk.chunk_id] = rrf_scores.get(chunk.chunk_id, 0.0) + ( | |
| self.sparse_weight / (self.rrf_k + rank + 1) | |
| ) | |
| # Sort by RRF score descending | |
| sorted_rrf = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)[:initial_top_k] | |
| for cid, score in sorted_rrf: | |
| candidate_map[cid] = chunk_lut[cid] | |
| candidate_scores[cid] = max(candidate_scores.get(cid, 0.0), score) | |
| # Prepare candidate list | |
| candidate_chunks = list(candidate_map.values()) | |
| # If mode is not reranking, sort by candidate scores and return | |
| if mode in ["dense_only", "bm25_only", "hybrid_no_rerank"]: | |
| sorted_candidates = sorted( | |
| candidate_chunks, | |
| key=lambda c: candidate_scores.get(c.chunk_id, 0.0), | |
| reverse=True | |
| )[:rerank_top_k] | |
| return { | |
| "query": query, | |
| "rewritten_query": rewritten_query, | |
| "sub_queries": sub_queries, | |
| "chunks": sorted_candidates, | |
| "scores": [candidate_scores.get(c.chunk_id, 0.0) for c in sorted_candidates], | |
| "mode": mode, | |
| } | |
| # Step 4: Cross-Encoder Reranking | |
| if not candidate_chunks: | |
| return { | |
| "query": query, | |
| "rewritten_query": rewritten_query, | |
| "sub_queries": sub_queries, | |
| "chunks": [], | |
| "scores": [], | |
| "mode": mode, | |
| } | |
| # Build cross-encoder input pairs: (rewritten_query, chunk.text) | |
| pairs = [[rewritten_query, c.text] for c in candidate_chunks] | |
| rerank_scores = self.reranker.predict(pairs) | |
| # Sort candidate chunks by cross-encoder score descending | |
| ranked_indices = np.argsort(rerank_scores)[::-1][:rerank_top_k] | |
| top_chunks = [candidate_chunks[i] for i in ranked_indices] | |
| top_scores = [float(rerank_scores[i]) for i in ranked_indices] | |
| return { | |
| "query": query, | |
| "rewritten_query": rewritten_query, | |
| "sub_queries": sub_queries, | |
| "chunks": top_chunks, | |
| "scores": top_scores, | |
| "mode": mode, | |
| } | |