add rag feature

multi_llm_chatbot_backend/app/api/routes.py

@@ -8,6 +8,7 @@ from app.llm.improved_ollama_client import ImprovedOllamaClient
 from app.models.persona import Persona
 from app.core.improved_orchestrator import ImprovedChatOrchestrator
 from app.core.session_manager import get_session_manager
+from app.core.rag_manager import get_rag_manager
 from app.models.default_personas import get_default_personas
 from app.utils.document_extractor import extract_text_from_file
 from app.utils.file_limits import is_within_upload_limit
@@ -182,64 +183,38 @@ async def switch_provider(provider_data: ProviderSwitch):
 # Main chat endpoint (SAME INTERFACE, improved backend)
 @router.post("/chat-sequential")
 async def chat_sequential(message: ChatMessage, request: Request):
-    """
-    SAME INTERFACE AS BEFORE - Generate advisor responses 
-    Now with improved session management behind the scenes
-    """
+    """Generate advisor responses - ENHANCED with RAG"""
     try:
-        # Get session using compatibility layer
         session_id = get_or_create_session_for_request(request, message.session_id)
         
-        # Use the new orchestrator with session management
+        # Process message through improved orchestrator (now with RAG)
         result = await chat_orchestrator.process_message(
             user_input=message.user_input,
             session_id=session_id,
-            response_length=message.response_length
+            response_length=message.response_length or "medium"
         )
         
-        # Convert new format back to old format for backward compatibility
-        if result["type"] == "clarification":
-            return {
-                "type": "orchestrator_question",
-                "responses": [{
-                    "persona": "PhD Advisor Assistant",
-                    "response": result["message"]
-                }],
-                "collected_info": {}
+        # Add RAG information to response
+        if result["type"] == "persona_responses":
+            # Count how many personas used documents
+            personas_with_docs = sum(1 for r in result["responses"] if r.get("used_documents", False))
+            total_chunks_used = sum(r.get("document_chunks_used", 0) for r in result["responses"])
+            
+            result["rag_info"] = {
+                "personas_using_documents": personas_with_docs,
+                "total_document_chunks_used": total_chunks_used,
+                "rag_enabled": True
             }
         
-        elif result["type"] == "persona_responses":
-            # Convert new response format to old format
-            return {
-                "type": "sequential_responses", 
-                "responses": [
-                    {
-                        "persona": resp["persona_name"],
-                        "persona_id": resp["persona_id"],
-                        "response": resp["response"]
-                    }
-                    for resp in result["responses"]
-                ],
-                "collected_info": {}
-            }
+        return result
         
-        else:
-            return {
-                "type": "error",
-                "responses": [{
-                    "persona": "System",
-                    "response": result.get("message", "Please try again.")
-                }]
-            }
-            
     except Exception as e:
-        logger.error(f"Error in chat_sequential: {e}")
+        logger.error(f"Error in chat-sequential: {str(e)}")
         return {
             "type": "error",
-            "responses": [{
-                "persona": "System",
-                "response": "I'm having trouble processing your request. Could you please try again?"
-            }]
+            "message": "I encountered an error processing your request. Please try again.",
+            "error": str(e),
+            "rag_enabled": False
         }
 
 # Individual advisor endpoint (SAME INTERFACE)
@@ -326,13 +301,21 @@ async def reply_to_advisor(reply: ReplyToAdvisor, request: Request):
             "response": "I'm having trouble generating a reply right now. Please try again."
         }
 
-# Document upload (SAME INTERFACE)
 @router.post("/upload-document")
 async def upload_document(file: UploadFile = File(...), request: Request = None):
-    """Upload document - SAME INTERFACE"""
+    """
+    Upload document with RAG integration - ENHANCED VERSION
+    
+    Now documents are:
+    1. Chunked intelligently 
+    2. Embedded and stored in ChromaDB
+    3. Available for semantic retrieval
+    4. NOT stored in session context (saves memory)
+    """
+    # Validate file type
     if file.content_type not in [
         "application/pdf",
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 
         "text/plain"
     ]:
         raise HTTPException(status_code=400, detail="Unsupported file type.")
@@ -347,16 +330,49 @@ async def upload_document(file: UploadFile = File(...), request: Request = None)
         # Simple size check
         MAX_FILE_SIZE = 10 * 1024 * 1024  # 10MB
         if len(file_bytes) > MAX_FILE_SIZE:
-            raise HTTPException(status_code=400, detail="Upload exceeds session document size limit (10 MB).")
+            raise HTTPException(status_code=400, detail="Upload exceeds file size limit (10 MB).")
 
+        # Extract text content
         content = extract_text_from_file(file_bytes, file.content_type)
         if not content.strip():
             raise HTTPException(status_code=400, detail="Document is empty or unreadable.")
 
-        # Add to session context using new system
-        session.add_uploaded_file(file.filename, content, len(file_bytes))
+        # Get RAG manager
+        rag_manager = get_rag_manager()
+        
+        # Determine file type for metadata
+        file_type_map = {
+            "application/pdf": "pdf",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+            "text/plain": "txt"
+        }
+        file_type = file_type_map.get(file.content_type, "unknown")
+        
+        # Add document to vector database instead of session context
+        rag_result = rag_manager.add_document(
+            content=content,
+            filename=file.filename,
+            session_id=session_id,
+            file_type=file_type
+        )
+        
+        if not rag_result["success"]:
+            raise HTTPException(status_code=500, detail=f"Failed to process document: {rag_result.get('error', 'Unknown error')}")
+        
+        # Add just the filename to session for tracking (not the full content)
+        session.uploaded_files.append(file.filename)
+        session.total_upload_size += len(file_bytes)
+        
+        # Add a brief document reference to session messages (not full content)
+        session.append_message("system", f"Document uploaded: {file.filename} ({rag_result['chunks_created']} chunks, {rag_result['total_tokens']} tokens)")
 
-        return {"message": "Document uploaded and added to context successfully."}
+        return {
+            "message": "Document uploaded and processed successfully.",
+            "filename": file.filename,
+            "chunks_created": rag_result["chunks_created"],
+            "total_tokens": rag_result["total_tokens"],
+            "processing_method": "RAG_vector_storage"
+        }
 
     except HTTPException:
         raise
@@ -364,6 +380,21 @@ async def upload_document(file: UploadFile = File(...), request: Request = None)
         logger.error(f"Error uploading document: {str(e)}")
         raise HTTPException(status_code=500, detail=f"Error processing document: {str(e)}")
 
+# Add new endpoint to get document statistics
+@router.get("/document-stats")
+async def get_document_stats(request: Request):
+    """Get statistics about uploaded documents in vector database"""
+    try:
+        session_id = get_or_create_session_for_request(request)
+        rag_manager = get_rag_manager()
+        
+        stats = rag_manager.get_document_stats(session_id)
+        return stats
+        
+    except Exception as e:
+        logger.error(f"Error getting document stats: {str(e)}")
+        return {"total_chunks": 0, "total_documents": 0, "documents": []}
+
 # Get uploaded files (SAME INTERFACE)
 @router.get("/uploaded-files")
 async def get_uploaded_filenames(request: Request):
@@ -379,31 +410,44 @@ async def get_uploaded_filenames(request: Request):
 # Context endpoint (SAME INTERFACE)
 @router.get("/context")
 async def get_context(request: Request):
-    """Get context - SAME INTERFACE"""
+    """Get context - ENHANCED with RAG information"""
     try:
         session_id = get_or_create_session_for_request(request)
         session = session_manager.get_session(session_id)
-        return session.messages  # Return messages in same format as before
+
+        # Get RAG statistics
+        rag_stats = session.get_rag_stats()
+        
+        return {
+            "messages": session.messages,
+            "rag_info": {
+                "total_documents": rag_stats.get("total_documents", 0),
+                "total_chunks": rag_stats.get("total_chunks", 0),
+                "documents": rag_stats.get("documents", [])
+            }
+        }
     except Exception as e:
         logger.error(f"Error getting context: {str(e)}")
-        return []
+        return {"messages": [], "rag_info": {"total_documents": 0, "total_chunks": 0}}
 
-# Reset session (SAME INTERFACE) 
 @router.post("/reset-session")
 async def reset_session(request: Request):
-    """Reset session - SAME INTERFACE"""
+    """Reset session - ENHANCED with RAG cleanup"""
     try:
         session_id = get_or_create_session_for_request(request)
-        success = chat_orchestrator.reset_session(session_id)
+        
+        # Use the enhanced reset that clears both conversation and vector DB
+        success = session_manager.reset_session_completely(session_id)
         
         if success:
-            return {"status": "reset", "message": "Session reset successfully"}
+            return {"status": "reset", "message": "Session and all documents reset successfully"}
         else:
             return {"status": "error", "message": "Failed to reset session"}
     except Exception as e:
         logger.error(f"Error resetting session: {e}")
         return {"status": "error", "message": "Failed to reset session"}
 
+
 # Legacy model endpoints (SAME INTERFACE)
 @router.post("/switch-model")
 async def switch_model(model_name: str = Body(...)):
@@ -422,30 +466,195 @@ async def get_current_model():
         "provider": current_provider
     }
 
-# Debug endpoint (SAME INTERFACE)
+@router.post("/search-documents")
+async def search_documents(request: Request, query: str = Body(..., embed=True), persona: str = Body("", embed=True)):
+    """
+    Search uploaded documents using RAG
+    
+    This endpoint allows direct document search for debugging/testing
+    """
+    try:
+        session_id = get_or_create_session_for_request(request)
+        rag_manager = get_rag_manager()
+        
+        # Get persona context for search enhancement
+        persona_contexts = {
+            "methodist": "methodology research design analysis",
+            "theorist": "theory theoretical framework conceptual",
+            "pragmatist": "practical application implementation"
+        }
+        persona_context = persona_contexts.get(persona, "")
+        
+        # Search documents
+        results = rag_manager.search_documents(
+            query=query,
+            session_id=session_id,
+            persona_context=persona_context,
+            n_results=5
+        )
+        
+        return {
+            "query": query,
+            "persona_filter": persona,
+            "results_count": len(results),
+            "results": results
+        }
+        
+    except Exception as e:
+        logger.error(f"Error searching documents: {str(e)}")
+        return {"query": query, "results_count": 0, "results": [], "error": str(e)}
+
+@router.get("/session-stats")
+async def get_session_stats(request: Request):
+    """Get comprehensive session statistics including RAG data"""
+    try:
+        session_id = get_or_create_session_for_request(request)
+        stats = session_manager.get_session_stats(session_id)
+        return stats
+    except Exception as e:
+        logger.error(f"Error getting session stats: {str(e)}")
+        return {"error": str(e)}
+
+
 @router.get("/debug/personas")
 async def debug_personas(request: Request):
-    """Debug personas - SAME INTERFACE"""
+    """Debug personas - ENHANCED with RAG information"""
     try:
         session_id = get_or_create_session_for_request(request)
         session = session_manager.get_session(session_id)
         
+        # Get RAG statistics
+        rag_manager = get_rag_manager()
+        rag_stats = rag_manager.get_document_stats(session_id)
+        
         return {
             "personas": {
                 pid: {
                     "name": persona.name,
-                    "prompt": persona.system_prompt[:100] + "..."
+                    "prompt": persona.system_prompt[:100] + "...",
+                    "retrieval_keywords": chat_orchestrator._get_persona_context_keywords(pid)
                 } for pid, persona in chat_orchestrator.personas.items()
             },
-            "context_length": len(session.messages),
-            "current_provider": current_provider
+            "session_info": {
+                "context_length": len(session.messages),
+                "uploaded_files": session.uploaded_files,
+                "rag_stats": rag_stats
+            },
+            "current_provider": current_provider,
+            "rag_enabled": True
         }
     except Exception as e:
         logger.error(f"Error in debug endpoint: {str(e)}")
         return {
             "personas": {},
-            "context_length": 0,
-            "current_provider": current_provider
+            "session_info": {"context_length": 0},
+            "current_provider": current_provider,
+            "rag_enabled": False,
+            "error": str(e)
+        }
+
+@router.post("/chat/{persona_id}")
+async def chat_with_specific_persona(persona_id: str, message: ChatMessage, request: Request):
+    """
+    Chat with a specific persona - Enhanced with RAG debugging
+    
+    This endpoint helps debug RAG integration by testing individual personas
+    """
+    try:
+        session_id = get_or_create_session_for_request(request, message.session_id)
+        
+        # Validate persona exists
+        if persona_id not in chat_orchestrator.personas:
+            available_personas = list(chat_orchestrator.personas.keys())
+            raise HTTPException(
+                status_code=400, 
+                detail=f"Persona '{persona_id}' not found. Available: {available_personas}"
+            )
+        
+        # Use the enhanced orchestrator method
+        result = await chat_orchestrator.chat_with_persona(
+            user_input=message.user_input,
+            persona_id=persona_id,
+            session_id=session_id,
+            response_length=message.response_length or "medium"
+        )
+        
+        # Fix: Handle the response structure properly
+        if result.get("type") == "single_persona_response" and "persona" in result:
+            persona_data = result["persona"]
+            
+            # Add debugging information
+            result["debug_info"] = {
+                "persona_id": persona_id,
+                "session_id": session_id,
+                "query_length": len(message.user_input),
+                "rag_manager_available": True,
+                "used_documents": persona_data.get("used_documents", False),
+                "chunks_used": persona_data.get("document_chunks_used", 0)
+            }
+        
+        return result
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error in individual persona chat: {str(e)}")
+        return {
+            "type": "error",
+            "message": f"Error chatting with {persona_id}: {str(e)}",
+            "persona_id": persona_id
+        }
+
+# Also add a debug endpoint to check RAG status:
+
+@router.get("/debug/rag-status")
+async def debug_rag_status(request: Request):
+    """
+    Debug endpoint to check RAG system status
+    """
+    try:
+        session_id = get_or_create_session_for_request(request)
+        
+        # Get RAG manager
+        rag_manager = get_rag_manager()
+        
+        # Get session stats
+        session_stats = session_manager.get_session_stats(session_id)
+        
+        # Test a simple search
+        test_search = rag_manager.search_documents(
+            query="test methodology research",
+            session_id=session_id,
+            persona_context="",
+            n_results=3
+        )
+        
+        return {
+            "rag_manager_healthy": True,
+            "session_id": session_id,
+            "session_stats": session_stats.get("rag_stats", {}),
+            "test_search_results": len(test_search),
+            "test_search_details": [
+                {
+                    "relevance": chunk.get("relevance_score", 0),
+                    "distance": chunk.get("distance", "unknown"),
+                    "text_length": len(chunk.get("text", "")),
+                    "filename": chunk.get("metadata", {}).get("filename", "unknown")
+                }
+                for chunk in test_search[:3]
+            ],
+            "persona_keywords": {
+                pid: chat_orchestrator._get_persona_context_keywords(pid)
+                for pid in chat_orchestrator.personas.keys()
+            }
+        }
+        
+    except Exception as e:
+        logger.error(f"Error in RAG debug: {str(e)}")
+        return {
+            "rag_manager_healthy": False,
+            "error": str(e),
+            "session_id": session_id if 'session_id' in locals() else "unknown"
         }
 
 # Ask endpoint (SAME INTERFACE)
@@ -476,6 +685,8 @@ async def ask_question(query: PersonaQuery, request: Request):
     except Exception as e:
         logger.error(f"Error in ask endpoint: {str(e)}")
         return {"response": "I encountered an error. Please try again."}
+    
+
 
 # Root endpoint (SAME INTERFACE)
 @router.get("/")

multi_llm_chatbot_backend/app/core/improved_orchestrator.py

@@ -2,6 +2,7 @@ from typing import Dict, List, Optional, Any
 from app.models.persona import Persona
 from app.core.session_manager import ConversationContext, get_session_manager
 from app.core.context_manager import get_context_manager
+from app.core.rag_manager import get_rag_manager
 from app.llm.llm_client import LLMClient
 import logging
 import re
@@ -85,7 +86,7 @@ class ImprovedChatOrchestrator:
                                session_id: Optional[str] = None,
                                response_length: str = "medium") -> Dict[str, Any]:
         """
-        Chat with a specific persona
+        Chat with a specific persona - ENHANCED WITH RAG
         """
         try:
             if persona_id not in self.personas:
@@ -100,7 +101,7 @@ class ImprovedChatOrchestrator:
             # Add user message
             session.append_message("user", user_input)
             
-            # Generate response from specific persona
+            # Generate response from specific persona with RAG
             persona = self.personas[persona_id]
             response = await self._generate_single_persona_response(session, persona, response_length)
             
@@ -111,7 +112,11 @@ class ImprovedChatOrchestrator:
                 "type": "single_persona_response",
                 "session_id": session.session_id,
                 "persona": response,
-                "context_summary": self.context_manager.get_context_summary(session.messages)
+                "context_summary": self.context_manager.get_context_summary(session.messages),
+                "rag_info": {
+                    "used_documents": response.get("used_documents", False),
+                    "document_chunks_used": response.get("document_chunks_used", 0)
+                }
             }
             
         except Exception as e:
@@ -123,38 +128,30 @@ class ImprovedChatOrchestrator:
                 "error": str(e)
             }
     
-    def _needs_clarification(self, session: ConversationContext, user_input: str) -> bool:
+    def _needs_clarification(self, session, user_input: str) -> bool:
         """
-        Determine if user input needs clarification
+        Determine if we need clarification - SIMPLIFIED VERSION
         """
-        # Don't ask for clarification if this is a follow-up message
-        user_messages = session.get_messages_by_role("user")
-        if len(user_messages) > 1:
-            return False
-        
-        # Check if input is vague
-        vague_patterns = [
-            r"i'm (not sure|unsure|confused|lost)",
-            r"i (don't know|dunno) (what|how|where)",
-            r"help me with (my|the|a) (thesis|research|phd)",
-            r"(what should i|how do i|where do i start)",
-            r"i need (help|advice|guidance)$",
-            r"(stuck|struggling) with",
-            r"(any|some) (advice|suggestions|ideas)$",
-            r"^(help|advice|guidance)$",
-        ]
-        
-        user_lower = user_input.lower().strip()
-        
-        for pattern in vague_patterns:
-            if re.search(pattern, user_lower):
+        try:
+            # Simple heuristics for clarification
+            if len(user_input.strip()) < 10:
                 return True
-        
-        # Check if input is too short (likely vague)
-        if len(user_input.split()) < 8:
-            return True
-        
-        return False
+            
+            # Check if it's a greeting or very general question
+            greeting_words = ['hi', 'hello', 'hey', 'good morning', 'good afternoon']
+            if any(word in user_input.lower() for word in greeting_words):
+                return True
+            
+            # Very vague questions
+            vague_words = ['help', 'advice', 'what should i do', 'i need help']
+            if len([word for word in vague_words if word in user_input.lower()]) > 0 and len(user_input.split()) < 8:
+                return True
+            
+            return False
+            
+        except Exception as e:
+            logger.error(f"Error in clarification check: {str(e)}")
+            return False  # Default to no clarification needed
     
     async def _generate_clarification_question(self, session: ConversationContext) -> str:
         """
@@ -196,62 +193,84 @@ class ImprovedChatOrchestrator:
         else:
             return "Could you provide more details about your specific situation or question?"
     
-    async def _generate_persona_responses(self, 
-                                        session: ConversationContext, 
-                                        response_length: str) -> List[Dict[str, Any]]:
+    async def _generate_persona_responses(self, session, response_length: str = "medium"):
         """
-        Generate responses from all personas
+        Generate responses from all personas with RAG integration
+        
+        Each persona gets persona-specific document retrieval for more targeted responses
         """
         responses = []
         
-        # Get the conversation context for personas
-        context_messages = session.get_recent_messages(limit=20)
-        
         for persona_id, persona in self.personas.items():
-            try:
-                response_data = await self._generate_single_persona_response(
-                    session, persona, response_length
-                )
-                responses.append(response_data)
-                
-                # Add persona response to session
-                session.append_message(persona_id, response_data['response'])
-                
-            except Exception as e:
-                logger.error(f"Error generating response for persona {persona_id}: {str(e)}")
-                # Add fallback response
-                responses.append({
-                    "persona_id": persona_id,
-                    "persona_name": persona.name,
-                    "response": self._get_fallback_response(persona_id),
-                    "error": True
-                })
+            logger.info(f"Generating response for {persona_id} with RAG")
+            
+            # Generate persona response with RAG
+            response_data = await self._generate_single_persona_response(session, persona, response_length)
+            
+            # Add persona response to session context
+            session.append_message(persona_id, response_data["response"])
+            
+            responses.append(response_data)
         
         return responses
     
-    async def _generate_single_persona_response(self, 
-                                              session: ConversationContext,
-                                              persona: Persona, 
-                                              response_length: str) -> Dict[str, Any]:
+    async def _generate_single_persona_response(self, session, persona, response_length: str = "medium"):
         """
-        Generate response from a single persona
+        Generate response from a single persona with RAG-enhanced context - FIXED VERSION
         """
         try:
-            # Get conversation context
-            context_messages = session.get_recent_messages(limit=20)
+            # Get conversation context (recent messages only for efficiency)
+            recent_messages = session.messages[-5:] if len(session.messages) > 5 else session.messages
+            
+            # Get the user's latest message for document retrieval
+            user_message = ""
+            try:
+                # Use the new method to get latest user message
+                user_message = session.get_latest_user_message() or ""
+            except AttributeError:
+                # Fallback: manually find latest user message
+                for msg in reversed(recent_messages):
+                    if msg.get('role') == 'user':
+                        user_message = msg.get('content', '')
+                        break
+            
+            # Retrieve relevant document context using RAG
+            document_context = ""
+            if user_message:
+                document_context = await self._retrieve_relevant_documents(
+                    user_input=user_message,
+                    session_id=session.session_id,
+                    persona_id=persona.id
+                )
+            
+            # Build enhanced context for the LLM
+            enhanced_context = []
+            
+            # Add document context at the beginning if available
+            if document_context:
+                enhanced_context.append({
+                    "role": "system",
+                    "content": f"{document_context}Based on the above document context and conversation history, please respond as {persona.name}."
+                })
             
-            # Generate response using persona
-            response = await persona.respond(context_messages, response_length)
+            # Add recent conversation messages
+            enhanced_context.extend(recent_messages)
+            
+            # Generate response with enhanced context
+            response = await persona.respond(enhanced_context, response_length)
             
             # Validate response
             if not self._is_valid_response(response, persona.id):
+                logger.warning(f"Invalid response from {persona.id}, using fallback")
                 response = self._get_fallback_response(persona.id)
             
             return {
                 "persona_id": persona.id,
                 "persona_name": persona.name,
                 "response": response,
-                "error": False
+                "error": False,
+                "used_documents": bool(document_context),
+                "document_chunks_used": document_context.count("[Document") if document_context else 0
             }
             
         except Exception as e:
@@ -260,7 +279,9 @@ class ImprovedChatOrchestrator:
                 "persona_id": persona.id,
                 "persona_name": persona.name,
                 "response": self._get_fallback_response(persona.id),
-                "error": True
+                "error": True,
+                "used_documents": False,
+                "document_chunks_used": 0
             }
     
     def _is_valid_response(self, response: str, persona_id: str) -> bool:
@@ -314,4 +335,73 @@ class ImprovedChatOrchestrator:
     
     def delete_session(self, session_id: str) -> bool:
         """Delete a session completely"""
-        return self.session_manager.delete_session(session_id)
+        return self.session_manager.delete_session(session_id)
+    
+    def _get_persona_context_keywords(self, persona_id: str) -> str:
+        """
+        Get persona-specific keywords for enhanced document retrieval
+        
+        This ensures each advisor gets the most relevant document chunks
+        based on their specialization
+        """
+        persona_keywords = {
+            "methodist": "methodology research design analysis methods data collection sampling validity reliability statistical approach quantitative qualitative",
+            "theorist": "theory theoretical framework conceptual model literature review philosophy epistemology ontology paradigm abstract concepts",
+            "pragmatist": "practical application implementation action steps next steps recommendation solution strategy timeline concrete advice"
+        }
+        return persona_keywords.get(persona_id, "")
+    
+    async def _retrieve_relevant_documents(self, user_input: str, session_id: str, persona_id: str = "") -> str:
+        """
+        Retrieve relevant document chunks using RAG - FIXED VERSION
+        """
+        try:
+            rag_manager = get_rag_manager()
+            
+            # Get persona-specific context for better retrieval
+            persona_context = self._get_persona_context_keywords(persona_id)
+            
+            # Search for relevant chunks
+            relevant_chunks = rag_manager.search_documents(
+                query=user_input,
+                session_id=session_id,
+                persona_context=persona_context,
+                n_results=3  # Limit to top 3 most relevant chunks
+            )
+            
+            logger.info(f"Retrieved {len(relevant_chunks)} total chunks for {persona_id}")
+            
+            if not relevant_chunks:
+                logger.info(f"No relevant documents found for query: {user_input[:50]}...")
+                return ""
+            
+            # Format retrieved content - LOWERED THRESHOLD
+            context_parts = []
+            for i, chunk in enumerate(relevant_chunks, 1):
+                filename = chunk["metadata"].get("filename", "Unknown")
+                relevance = chunk["relevance_score"]
+                distance = chunk.get("distance", "unknown")
+                text = chunk["text"]
+                
+                logger.info(f"Chunk {i} for {persona_id}: relevance={relevance:.3f}, distance={distance}")
+                
+                # LOWERED threshold from 0.3 to 0.1 to allow more chunks through
+                if relevance > 0.1:
+                    context_parts.append(
+                        f"[Document {i}: {filename} (relevance: {relevance:.3f})]\n{text[:500]}..."  # Limit text length
+                    )
+                    logger.info(f"Including chunk {i} for {persona_id} (relevance: {relevance:.3f})")
+                else:
+                    logger.info(f"Excluding chunk {i} for {persona_id} (relevance too low: {relevance:.3f})")
+            
+            if context_parts:
+                retrieved_context = "\n\n".join(context_parts)
+                logger.info(f"Using {len(context_parts)} relevant chunks for {persona_id}")
+                return f"RELEVANT DOCUMENT CONTEXT:\n{retrieved_context}\n\n"
+            
+            logger.info(f"No chunks met relevance threshold for {persona_id}")
+            return ""
+            
+        except Exception as e:
+            logger.error(f"Error retrieving documents for {persona_id}: {str(e)}")
+            return ""

multi_llm_chatbot_backend/app/core/session_manager.py

@@ -4,49 +4,106 @@ import uuid
 from dataclasses import dataclass, field
 import asyncio
 from threading import Lock
+from app.core.rag_manager import get_rag_manager
 
 @dataclass
 class ConversationContext:
-    """Individual conversation context for a session"""
-    session_id: str
-    messages: List[dict] = field(default_factory=list)
-    uploaded_files: List[str] = field(default_factory=list)
-    total_upload_size: int = 0
-    metadata: Dict[str, Any] = field(default_factory=dict)
-    created_at: datetime = field(default_factory=datetime.now)
-    last_accessed: datetime = field(default_factory=datetime.now)
+    """Enhanced conversation context for RAG integration"""
     
+    def __init__(self, session_id: str = None):
+        self.session_id = session_id or str(uuid.uuid4())
+        self.messages: List[Dict[str, str]] = []
+        self.uploaded_files: List[str] = []  # Now just stores filenames, not content
+        self.total_upload_size: int = 0  # For tracking purposes only
+        self.created_at = datetime.now()
+        self.last_accessed = datetime.now()
+        
+        # New RAG-related attributes
+        self.document_chunks_count: int = 0  # Track total chunks in vector DB
+        self.last_retrieval_stats: Dict[str, Any] = {}  # Last RAG retrieval info
+
     def append_message(self, role: str, content: str):
-        """Add a message to the conversation"""
+        """Add a message to the conversation history"""
         self.messages.append({
             "role": role,
             "content": content,
             "timestamp": datetime.now().isoformat()
         })
         self.last_accessed = datetime.now()
-    
-    def get_recent_messages(self, limit: int = 20) -> List[dict]:
-        """Get recent messages with limit"""
-        return self.messages[-limit:] if len(self.messages) > limit else self.messages
-    
-    def get_messages_by_role(self, role: str) -> List[dict]:
-        """Get all messages from a specific role"""
-        return [msg for msg in self.messages if msg['role'] == role]
-    
+
     def clear_messages(self):
-        """Clear all messages but preserve metadata"""
-        self.messages = []
+        """Clear conversation messages but keep document references"""
+        self.messages.clear()
         self.last_accessed = datetime.now()
+
+    def get_messages_by_role(self, role: str) -> List[Dict[str, str]]:
+        """Get all messages by a specific role - ADDED METHOD"""
+        return [msg for msg in self.messages if msg.get('role') == role]
     
+    def get_recent_messages(self, count: int = 10) -> List[Dict[str, str]]:
+        """Get the most recent N messages"""
+        return self.messages[-count:] if len(self.messages) > count else self.messages
+    
+    def get_user_messages(self) -> List[Dict[str, str]]:
+        """Get all user messages"""
+        return self.get_messages_by_role('user')
+    
+    def get_latest_user_message(self) -> Optional[str]:
+        """Get the content of the most recent user message"""
+        user_messages = self.get_user_messages()
+        return user_messages[-1]['content'] if user_messages else None
+
     def add_uploaded_file(self, filename: str, content: str, file_size: int):
-        """Add uploaded file content to context"""
+        """
+        Add uploaded file - MODIFIED FOR RAG
+        
+        Now we only track the filename and size in session context.
+        The actual content goes to the vector database via RAG manager.
+        """
         self.uploaded_files.append(filename)
         self.total_upload_size += file_size
-        self.append_message("document", f"[Uploaded: {filename}]\n{content}")
-    
+        
+        # Add a system message noting the upload (not the full content)
+        self.append_message("system", f"Document '{filename}' uploaded and processed into vector database")
+        
+        # Update document chunk count from RAG manager
+        try:
+            rag_manager = get_rag_manager()
+            stats = rag_manager.get_document_stats(self.session_id)
+            self.document_chunks_count = stats.get("total_chunks", 0)
+        except Exception as e:
+            print(f"Warning: Could not update chunk count: {e}")
+
     def get_context_size(self) -> int:
-        """Calculate total context size in characters"""
+        """Calculate conversation context size in characters (excluding vector DB documents)"""
         return sum(len(msg['content']) for msg in self.messages)
+    
+    def get_rag_stats(self) -> Dict[str, Any]:
+        """Get statistics about documents in vector database for this session"""
+        try:
+            rag_manager = get_rag_manager()
+            return rag_manager.get_document_stats(self.session_id)
+        except Exception as e:
+            return {"error": str(e), "total_chunks": 0, "total_documents": 0}
+
+    def clear_all_data(self):
+        """Clear both conversation and vector database documents"""
+        # Clear conversation messages
+        self.clear_messages()
+        
+        # Clear vector database documents
+        try:
+            rag_manager = get_rag_manager()
+            success = rag_manager.delete_session_documents(self.session_id)
+            if success:
+                self.uploaded_files.clear()
+                self.total_upload_size = 0
+                self.document_chunks_count = 0
+                self.append_message("system", "All conversation history and documents cleared")
+            else:
+                self.append_message("system", "Conversation cleared, but some documents may remain")
+        except Exception as e:
+            self.append_message("system", f"Conversation cleared, document cleanup failed: {str(e)}")
 
 class SessionManager:
     """Thread-safe session manager for handling multiple user conversations"""
@@ -115,6 +172,42 @@ class SessionManager:
         
         if expired_sessions:
             print(f"Cleaned up {len(expired_sessions)} expired sessions")
+    
+    def reset_session_completely(self, session_id: str) -> bool:
+        """
+        Completely reset a session - both conversation and vector database documents
+        """
+        with self.lock:
+            if session_id in self.sessions:
+                session = self.sessions[session_id]
+                session.clear_all_data()
+                return True
+            return False
+    
+    def get_session_stats(self, session_id: str) -> Dict[str, Any]:
+        """Get comprehensive session statistics including RAG data"""
+        with self.lock:
+            if session_id not in self.sessions:
+                return {"error": "Session not found"}
+            
+            session = self.sessions[session_id]
+            
+            # Get basic session stats
+            basic_stats = {
+                "session_id": session_id,
+                "message_count": len(session.messages),
+                "uploaded_files": session.uploaded_files,
+                "total_upload_size": session.total_upload_size,
+                "context_size_chars": session.get_context_size(),
+                "created_at": session.created_at.isoformat(),
+                "last_accessed": session.last_accessed.isoformat()
+            }
+            
+            # Get RAG stats
+            rag_stats = session.get_rag_stats()
+            
+            # Combine stats
+            return {**basic_stats, "rag_stats": rag_stats}
 
 # Global session manager instance
 session_manager = SessionManager()