feat: implement semantic chunking, dual-retrieval cross-lingual search, and direct Q&A bypass routing

config.py

@@ -6,5 +6,5 @@ EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
 CHUNK_SIZE = 500
 CHUNK_OVERLAP = 50
 MAX_TOKENS = 150
-TEMPERATURE = 0.7
+TEMPERATURE = 0.0
 USE_OLLAMA = os.getenv("USE_OLLAMA", "True").lower() == "true"

llm/inference.py

@@ -201,51 +201,103 @@ def generate_response(prompt: str, max_tokens: int = None) -> str:
 
 def transliterate_hinglish_to_hindi(text: str) -> str:
     """
-    Use the LLM to transliterate Romanized Hinglish to Devanagari Hindi.
+    Translate Hinglish to Devanagari Hindi by first translating to English
+    and then translating the English to Hindi. This is highly accurate and fast.
     """
-    prompt = (
-        "<|im_start|>system\n"
-        "You are a precise transliteration tool. Transliterate Romanized Hinglish (Latin script) into Devanagari Hindi script. "
-        "Keep English technical words as they are, but write them in Devanagari script (e.g. 'rules' -> 'रूल्स'). "
-        "Respond ONLY with the transliterated text. Do not add explanations or extra text.\n"
-        "<|im_end|>\n"
-        f"<|im_start|>user\n{text}<|im_end|>\n"
-        "<|im_start|>assistant\n"
-    )
     try:
-        # Transliteration targets a short text, limit max_tokens to 100 for safety
-        result = generate_response(prompt, max_tokens=100)
-        # Fallback if empty or failed
-        if not result.strip():
-            return text
-        return result
-    except Exception:
-        return text
+        from deep_translator import GoogleTranslator
+        # Translate to English first
+        translator_en = GoogleTranslator(source='auto', target='en')
+        english_text = translator_en.translate(text)
+        if english_text and english_text.strip():
+            # Translate English to Hindi
+            translator_hi = GoogleTranslator(source='en', target='hi')
+            hindi_text = translator_hi.translate(english_text)
+            if hindi_text and hindi_text.strip():
+                return hindi_text.strip()
+    except Exception as e:
+        print(f"[WARNING] Hinglish to Hindi translation chain failed: {e}")
+        
+    return text
 
 
 def transliterate_hindi_to_hinglish(text: str) -> str:
     """
-    Use the LLM to transliterate Devanagari Hindi to Romanized Hinglish.
+    Deterministically transliterate Devanagari Hindi text to Romanized Hinglish (Latin script).
+    Uses rule-based mappings to avoid LLM hallucinations, gibberish, and latency.
     """
-    prompt = (
-        "<|im_start|>system\n"
-        "You are a precise transliteration tool. Transliterate Devanagari Hindi script into Romanized Hinglish (Latin script).\n"
-        "Example:\n"
-        "Input: मेरी ताकत में टीमवर्क, संचार (communication) और समस्या समाधान (problem-solving) कौशल शामिल हैं।\n"
-        "Output: Meri strengths me teamwork, communication, aur problem-solving skills shamil hain.\n"
-        "Respond ONLY with the transliterated text. Do not add explanations.\n"
-        "<|im_end|>\n"
-        f"<|im_start|>user\nInput: {text}<|im_end|>\n"
-        "<|im_start|>assistant\n"
-    )
-    try:
-        # Transliteration targets a short text, limit max_tokens to 100 for safety
-        result = generate_response(prompt, max_tokens=100)
-        if not result.strip():
-            return text
-        return result
-    except Exception:
-        return text
+    vowels = {
+        'अ': 'a', 'आ': 'aa', 'इ': 'i', 'ई': 'ee', 'उ': 'u', 'ऊ': 'oo',
+        'ए': 'e', 'ऐ': 'ai', 'ओ': 'o', 'औ': 'au', 'ऋ': 'ri', 'एॅ': 'e', 'ऑ': 'o'
+    }
+    
+    vowel_signs = {
+        'ा': 'a', 'ि': 'i', 'ी': 'ee', 'ु': 'u', 'ू': 'oo',
+        'े': 'e', 'ै': 'ai', 'ो': 'o', 'ौ': 'au', 'ृ': 'ri',
+        'ॅ': 'e', 'ॉ': 'o', 'ं': 'n', 'ँ': 'n', 'ः': 'h'
+    }
+    
+    consonants = {
+        'क': 'k', 'ख': 'kh', 'ग': 'g', 'घ': 'gh', 'ङ': 'n',
+        'च': 'ch', 'छ': 'chh', 'ज': 'j', 'झ': 'jh', 'ञ': 'n',
+        'ट': 't', 'ठ': 'th', 'ड': 'd', 'ढ': 'dh', 'ण': 'n',
+        'त': 't', 'थ': 'th', 'द': 'd', 'ध': 'dh', 'न': 'n',
+        'प': 'p', 'फ': 'ph', 'ब': 'b', 'भ': 'bh', 'म': 'm',
+        'य': 'y', 'र': 'r', 'ल': 'l', 'व': 'v',
+        'श': 'sh', 'ष': 'sh', 'स': 's', 'ह': 'h',
+        'क्ष': 'ksh', 'त्र': 'tr', 'ज्ञ': 'gy', 'श्र': 'shr',
+        'ड़': 'd', 'ढ़': 'dh'
+    }
+    
+    result = []
+    i = 0
+    n = len(text)
+    
+    while i < n:
+        char = text[i]
+        
+        # Look ahead for conjuncts or dotted characters
+        if i + 1 < n and text[i:i+2] in consonants:
+            char = text[i:i+2]
+            i += 1
+        elif i + 1 < n and text[i+1] == '़': # nukta
+            char = text[i:i+2]
+            i += 1
+            
+        if char in vowels:
+            result.append(vowels[char])
+        elif char in consonants:
+            base = consonants[char]
+            has_modifier = False
+            if i + 1 < n:
+                next_char = text[i+1]
+                if next_char == '्':
+                    result.append(base)
+                    i += 1
+                    has_modifier = True
+                elif next_char in vowel_signs:
+                    result.append(base + vowel_signs[next_char])
+                    i += 1
+                    has_modifier = True
+            if not has_modifier:
+                is_end_of_word = True
+                if i + 1 < n:
+                    next_char = text[i+1]
+                    if '\u0900' <= next_char <= '\u097F':
+                        is_end_of_word = False
+                
+                if is_end_of_word:
+                    result.append(base)
+                else:
+                    result.append(base + 'a')
+        elif char in vowel_signs:
+            result.append(vowel_signs[char])
+        else:
+            result.append(char)
+            
+        i += 1
+        
+    return "".join(result)
 
 
 def translate_to_english(text: str) -> str:

rag/chunking.py

@@ -1,12 +1,64 @@
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from config import CHUNK_SIZE, CHUNK_OVERLAP
+import re
+from langchain_core.documents import Document
 
 def split_documents(documents):
-
-    splitter = RecursiveCharacterTextSplitter(
-        chunk_size=CHUNK_SIZE,
-        chunk_overlap=CHUNK_OVERLAP
-    )
-    chunks = splitter.split_documents(documents)
-
+    """
+    Splits documents semantically by separating each Q&A block and section/heading
+    into its own self-contained chunk. This ensures high vector search accuracy.
+    """
+    chunks = []
+    
+    for doc in documents:
+        content = doc.page_content
+        metadata = doc.metadata
+        
+        lines = content.split('\n')
+        
+        current_chunk_lines = []
+        in_qa = False
+        
+        for line in lines:
+            line_str = line.strip()
+            if not line_str:
+                continue
+                
+            # Detect new QA block starting
+            is_new_q = re.match(r'^(?:Q\d+\.|Question:|Q:|प्रश्न\s*\d*:|प्रश्न\s*\d+\.|प्र\s*\d+:)\s*(.*)', line_str, re.IGNORECASE)
+            
+            if is_new_q:
+                # Save previous chunk if any
+                if current_chunk_lines:
+                    chunks.append(Document(
+                        page_content="\n".join(current_chunk_lines),
+                        metadata=metadata.copy()
+                    ))
+                current_chunk_lines = [line_str]
+                in_qa = True
+                continue
+                
+            # Detect new major headings/sections (e.g. numbered headings)
+            is_heading = (
+                re.match(r'^\d+\.\s+[A-Z]', line_str) or 
+                (len(line_str) < 50 and line_str.isupper())
+            )
+            
+            if is_heading and in_qa:
+                # End Q&A context and start a new section chunk
+                if current_chunk_lines:
+                    chunks.append(Document(
+                        page_content="\n".join(current_chunk_lines),
+                        metadata=metadata.copy()
+                    ))
+                current_chunk_lines = [line_str]
+                in_qa = False
+                continue
+                
+            current_chunk_lines.append(line_str)
+            
+        if current_chunk_lines:
+            chunks.append(Document(
+                page_content="\n".join(current_chunk_lines),
+                metadata=metadata.copy()
+            ))
+            
     return chunks

rag/prompt_builder.py

@@ -1,4 +1,55 @@
+import re
+
+def format_context_as_xml(context: str) -> str:
+    """
+    Format raw context text containing QA pairs and plain text into structured XML blocks.
+    This ensures no information is discarded and helps small LLMs process documents cleanly.
+    """
+    lines = context.split('\n')
+    qa_pairs = []
+    plain_text_lines = []
+    current_q = None
+    current_a = []
+    
+    for line in lines:
+        line_str = line.strip()
+        if not line_str:
+            continue
+            
+        q_match = re.match(r'^(?:Q\d+\.|Question:|Q:)\s*(.*)', line_str, re.IGNORECASE)
+        if q_match:
+            if current_q and current_a:
+                qa_pairs.append((current_q, " ".join(current_a)))
+            current_q = q_match.group(1).strip()
+            current_a = []
+            continue
+            
+        a_match = re.match(r'^(?:Answer:|Ans:|A:)\s*(.*)', line_str, re.IGNORECASE)
+        if a_match:
+            current_a.append(a_match.group(1).strip())
+            continue
+            
+        if current_q and current_a:
+            current_a.append(line_str)
+        else:
+            plain_text_lines.append(line_str)
+            
+    if current_q and current_a:
+        qa_pairs.append((current_q, " ".join(current_a)))
+        
+    xml_parts = []
+    if plain_text_lines:
+        xml_parts.append("<information>\n" + "\n".join(plain_text_lines) + "\n</information>")
+        
+    for q, a in qa_pairs:
+        xml_parts.append(f"<qa>\n<question>{q}</question>\n<answer>{a}</answer>\n</qa>")
+        
+    return "\n".join(xml_parts)
+
+
 def build_prompt(context: str, question: str, language: str = "english"):
+    formatted_context = format_context_as_xml(context)
+    
     if language == "hindi":
         system_prompt = (
             "आप एक सहायक एआई सहायक हैं। नीचे दिए गए संदर्भ (context) से सीधे और संक्षिप्त उत्तर दें (3 वाक्यों से कम)।\n"
@@ -14,26 +65,21 @@ def build_prompt(context: str, question: str, language: str = "english"):
         )
     else:
         system_prompt = (
-            "You are a helpful AI assistant. Answer the user's question based ONLY on the provided context. Keep your answers brief and under 3 sentences.\n"
-            "If the answer is not available in the context, respond exactly with: 'Sorry, I can't help in this context.'\n\n"
-            "Example:\n"
-            "Context:\n"
-            "Q3. What are your strengths?\n"
-            "Answer: My strengths include teamwork, communication, and problem-solving skills.\n"
-            "Question: What are your strengths?\n"
-            "Response: My strengths include teamwork, communication, and problem-solving skills."
+            "You are a helpful AI assistant. Answer the user's question directly based on the XML qa blocks in the context. "
+            "Find the matching question and return its exact answer. Do not add meta-commentary or extra text.\n"
+            "If the answer is not available in the context, respond exactly with: 'Sorry, I can't help in this context.'"
         )
 
     prompt = f"""<|im_start|>system
 {system_prompt}<|im_end|>
 <|im_start|>user
 Context:
-{context}
+{formatted_context}
 
 Question:
 {question}<|im_end|>
 <|im_start|>assistant
-"""
+Answer: """
     return prompt
 
 

rag/rag_pipeline.py

@@ -10,14 +10,15 @@ GREETINGS = {
     "thank you", "thanks", "hell", "helo", "namaste", "pranam"
 }
 
-# Common Hinglish words (pronouns, auxiliary verbs, particles, and simple verbs)
+# Common Hinglish words (non-overlapping with standard English vocabulary to avoid false positives)
 HINGLISH_KEYWORDS = {
     "kya", "kyu", "kyoon", "kab", "kaha", "kahin", "kaise", "kaun", "kis",
-    "hai", "hain", "hoon", "ho", "tha", "the", "thi", "raha", "rahe", "rahi", "rha", "rhe", "rhi",
-    "mera", "meri", "mere", "mujhe", "apna", "apni", "apne", "tum", "tumhara", "tumhari", "aap", "aapka", "aapki", "aapke",
-    "hum", "humara", "humari", "se", "ko", "par", "pe", "ka", "ki", "ke", "liye", "bhi", "hi", "toh", "to", "aur", "ya", "lekin", "magar",
+    "hai", "hain", "hoon", "tha", "the", "thi", "raha", "rahe", "rahi", "rha", "rhe", "rhi",
+    "mera", "meri", "mere", "mujhe", "apna", "apni", "apne", "tum", "tumhara", "tumhari",
+    "aap", "aapka", "aapki", "aapke", "hum", "humara", "humari",
+    "liye", "lekin", "magar", "bhi", "toh",
     "karo", "kar", "karte", "karta", "karti", "kiya", "kr", "kra", "kro", "krna", "karna",
-    "batao", "bolo", "likho", "dikhao", "suno", "samjho", "chahiye", "de", "do", "le", "lo", "aao", "jaao", "jao"
+    "batao", "bolo", "likho", "dikhao", "suno", "samjho", "chahiye", "aao", "jaao", "jao"
 }
 
 
@@ -86,6 +87,75 @@ def _clean_response(text: str) -> str:
     return cleaned
 
 
+def query_matches_question(query: str, question: str) -> bool:
+    """
+    Check if query matches a target question by looking at keyword overlap.
+    Filters out common stop words to keep match focus on meaningful words.
+    """
+    def get_keywords(text: str):
+        text = re.sub(r'[^\w\s]', '', text.lower())
+        words = text.split()
+        stop_words = {"what", "why", "how", "who", "whom", "when", "where", "which", "is", "are", "do", "does", "you", "your", "we", "i", "the", "a", "an", "in", "on", "at", "to", "for", "of", "with", "about"}
+        return {w for w in words if w not in stop_words and len(w) > 2}
+        
+    q_words = get_keywords(query)
+    target_words = get_keywords(question)
+    
+    if not q_words or not target_words:
+        return False
+        
+    intersection = q_words.intersection(target_words)
+    if len(intersection) >= min(len(q_words), len(target_words)) * 0.5:
+        return True
+    return False
+
+
+def parse_context_hybrid(context: str):
+    """
+    Parse retrieved context into structured QA blocks and general plain text.
+    """
+    lines = context.split('\n')
+    qa_blocks = []
+    plain_text_lines = []
+    
+    current_q = None
+    current_a = []
+    
+    for line in lines:
+        line_str = line.strip()
+        if not line_str:
+            continue
+            
+        q_match = re.match(r'^(?:Q\d+\.|Question:|Q:|प्रश्न\s*\d*:|प्रश्न\s*\d+\.|प्र\s*\d+:)\s*(.*)', line_str, re.IGNORECASE)
+        if q_match:
+            if current_q and current_a:
+                qa_blocks.append({
+                    "question": current_q,
+                    "answer": " ".join(current_a)
+                })
+            current_q = q_match.group(1).strip()
+            current_a = []
+            continue
+            
+        a_match = re.match(r'^(?:Answer:|Ans:|A:|उत्तर:)\s*(.*)', line_str, re.IGNORECASE)
+        if a_match:
+            current_a.append(a_match.group(1).strip())
+            continue
+            
+        if current_q and current_a:
+            current_a.append(line_str)
+        else:
+            plain_text_lines.append(line_str)
+            
+    if current_q and current_a:
+        qa_blocks.append({
+            "question": current_q,
+            "answer": " ".join(current_a)
+        })
+        
+    return qa_blocks, "\n".join(plain_text_lines)
+
+
 def generate_rag_response(question: str):
     # 1. Detect query language
     lang = detect_language(question)
@@ -97,19 +167,21 @@ def generate_rag_response(question: str):
 
     # 3. Translate query to English if it is Hindi or Hinglish (as uploaded documents are in English)
     search_query = question
+    hindi_query = None
     if lang in ["hindi", "hinglish"]:
         search_query = translate_to_english(question)
+        from llm.inference import transliterate_hinglish_to_hindi
+        hindi_query = transliterate_hinglish_to_hindi(question)
 
     # 4. Clean text
     search_query = clean_text(search_query)
 
     # 5. Retrieve context using similarity search
     context = retrieve_context(search_query)
-
-    # Clean the retrieved context of QA interview formatting to avoid model grading/evaluation triggers
-    if context:
-        context = re.sub(r'(?i)Q\d+\.\s*', '', context)
-        context = re.sub(r'(?i)(Answer|Ans|A):\s*', '', context)
+    if lang in ["hindi", "hinglish"] and hindi_query:
+        context_hi = retrieve_context(clean_text(hindi_query))
+        if context_hi and context_hi.strip():
+            context = context + "\n" + context_hi
 
     # Fast check: If the retrieved context is empty/whitespace, return the localized fallback immediately
     if not context or not context.strip():
@@ -120,42 +192,101 @@ def generate_rag_response(question: str):
         else:
             return "Sorry, I can't help in this context."
 
-    # 6. Format prompt targeting English generation for high-quality fact retrieval
-    prompt = build_prompt(context, search_query, language="english")
+    # Parse hybrid context
+    qa_blocks, general_text = parse_context_hybrid(context)
+    
+    # Check if the query matches any QA block
+    matching_qa = None
+    for qa in qa_blocks:
+        if query_matches_question(search_query, qa["question"]):
+            matching_qa = qa
+            break
+        if hindi_query and query_matches_question(hindi_query, qa["question"]):
+            matching_qa = qa
+            break
+            
+    if matching_qa:
+        # Directly extract the answer string for exact Q&A matches.
+        # This completely bypasses LLM latency and hallucination risks.
+        response = matching_qa["answer"]
+    else:
+        if general_text.strip():
+            system_prompt = (
+                "You are a helpful AI assistant. Answer the user's question directly based ONLY on the provided context. "
+                "Keep your answer brief, factual, and under 3 sentences. Do not add meta-commentary or repeat the context.\n"
+                "If the answer is not available in the context, respond exactly with: 'Sorry, I can't help in this context.'"
+            )
+            prompt = f"""<|im_start|>system
+{system_prompt}<|im_end|>
+<|im_start|>user
+Context:
+{general_text}
 
-    # 7. Generate response in English
-    response = generate_response(prompt)
-    response = _clean_response(response)
+Question:
+{search_query}<|im_end|>
+<|im_start|>assistant
+"""
+            response = generate_response(prompt)
+            response = _clean_response(response)
+        else:
+            response = "Sorry, I can't help in this context."
 
     # 8. Post-process language-specific translation/fallback
-    if "sorry, i can't help" in response.lower() or "i don't know" in response.lower():
+    resp_lower = response.lower()
+    fallback_signals = ["can't help", "cannot help", "don't know", "do not know", "sorry", "not available", "not present", "no information"]
+    is_fallback = any(sig in resp_lower for sig in fallback_signals)
+
+    if is_fallback:
         if lang == "hindi":
             return "माफ़ कीजिए, मैं इस संदर्भ में आपकी मदद नहीं कर सकता।"
         elif lang == "hinglish":
             return "Sorry, main is context me help nahi kar sakta."
         else:
-            return response
+            return "Sorry, I can't help in this context."
+
+    # Robust multi-directional translation based on content language
+    has_hindi = any('\u0900' <= char <= '\u097F' for char in response)
 
-    if lang == "hindi":
-        try:
-            from deep_translator import GoogleTranslator
-            translated = GoogleTranslator(source='en', target='hi').translate(response)
-            if translated and translated.strip():
-                return translated.strip()
-        except Exception as e:
-            print(f"[WARNING] English to Hindi translation failed: {e}")
-
-    elif lang == "hinglish":
-        try:
-            from deep_translator import GoogleTranslator
-            # Translate to Hindi first for vocabulary alignment
-            translated_hi = GoogleTranslator(source='en', target='hi').translate(response)
-            if translated_hi and translated_hi.strip():
-                # Transliterate Hindi to Hinglish phonetically
-                hinglish_res = transliterate_hindi_to_hinglish(translated_hi)
+    if has_hindi:
+        if lang == "hindi":
+            return response
+        elif lang == "hinglish":
+            try:
+                hinglish_res = transliterate_hindi_to_hinglish(response)
                 if hinglish_res and hinglish_res.strip():
                     return hinglish_res.strip()
-        except Exception as e:
-            print(f"[WARNING] English to Hinglish translation/transliteration failed: {e}")
-
-    return response
+            except Exception as e:
+                print(f"[WARNING] Hindi to Hinglish transliteration failed: {e}")
+            return response
+        else:  # english
+            try:
+                from deep_translator import GoogleTranslator
+                translated = GoogleTranslator(source='hi', target='en').translate(response)
+                if translated and translated.strip():
+                    return translated.strip()
+            except Exception as e:
+                print(f"[WARNING] Hindi to English translation failed: {e}")
+            return response
+    else:
+        if lang == "hindi":
+            try:
+                from deep_translator import GoogleTranslator
+                translated = GoogleTranslator(source='en', target='hi').translate(response)
+                if translated and translated.strip():
+                    return translated.strip()
+            except Exception as e:
+                print(f"[WARNING] English to Hindi translation failed: {e}")
+        elif lang == "hinglish":
+            try:
+                from deep_translator import GoogleTranslator
+                # Translate to Hindi first for vocabulary alignment
+                translated_hi = GoogleTranslator(source='en', target='hi').translate(response)
+                if translated_hi and translated_hi.strip():
+                    # Transliterate Hindi to Hinglish phonetically
+                    hinglish_res = transliterate_hindi_to_hinglish(translated_hi)
+                    if hinglish_res and hinglish_res.strip():
+                        return hinglish_res.strip()
+            except Exception as e:
+                print(f"[WARNING] English to Hinglish translation/transliteration failed: {e}")
+                
+        return response