Improve backend logic around user input parsing (#12)

* fix: improve canvas confidence scores and insight extraction

Switch insight extraction from fragile JSON parsing to plain-text numbered list format. Add content-driven confidence variability: LLM-extracted insights score 0.75-0.90 based on length, rule-based insights score 0.55-0.70 based on pattern match count.

Made-with: Cursor

* fix: improve orchestrator clarification and LLM response handling

Rewrite _needs_clarification to check keywords before length, preventing false triggers. Add generate_contextual_clarification using LLM for tailored follow-up questions. Wire clarification check into /api/chat. Filter Gemini 2.5 thinking-mode parts from responses. Clean up _clean_response in both Gemini and Ollama clients to preserve Markdown.

Made-with: Cursor

* Address issues noted in review

* Revert JSON prompting and clean up imports

* Refactor input sanitization to log malformed input requests

---------

Co-authored-by: NeonClary <askclary@gmail.com>

multi_llm_chatbot_backend/app/api/routes/chat.py

@@ -194,9 +194,32 @@ async def chat_sequential_enhanced(
         rag_stats = session.get_rag_stats()
         logger.info(f"Session {session_id} has {rag_stats.get('total_documents', 0)} documents available")
         
-        # Add user message to session (needed for persona ranking)
+        # Warn if a repeated input message is received
+        if all((
+            session.messages,
+            session.messages[-1].get('role') == 'user',
+            session.messages[-1].get('content') == message.user_input
+            )):
+            # TODO: This should be handled in the front-end input
+            logger.warning(f"Repeated user input: {message.user_input}")
         session.append_message("user", message.user_input)
         
+        # Check if the user's message is vague and needs clarification
+        if chat_orchestrator._needs_clarification(session, message.user_input):
+            clarification = await chat_orchestrator.generate_contextual_clarification(
+                message.user_input
+            )
+            logger.info(f"Clarification triggered for input: {message.user_input!r}")
+            return {
+                "status": "clarification_needed",
+                "message": clarification["question"],
+                "suggestions": clarification["suggestions"],
+                "session_debug": {
+                    "session_id": session_id,
+                    "trigger": "vague_input"
+                }
+            }
+        
         # RESTORED: Get intelligently ordered personas based on context
         top_personas = await chat_orchestrator.get_top_personas(
             session_id=session_id, 
@@ -434,4 +457,4 @@ async def ask_question(query: PersonaQuery, request: Request):
         
     except Exception as e:
         logger.error(f"Error in ask endpoint: {str(e)}")
-        return {"response": "I encountered an error. Please try again."}
+        return {"response": "I encountered an error. Please try again."}

multi_llm_chatbot_backend/app/config.py

@@ -101,8 +101,16 @@ class PersonasConfig(BaseModel):
 class OrchestratorConfig(BaseModel):
     min_words_without_keywords: int = 6
     specific_keywords: List[str] = []
-    clarification_questions: List[str] = []
-    clarification_suggestions: List[str] = []
+    clarification_questions: List[str] = [
+            "Could you provide more details about what you need help with?"]
+    clarification_suggestions: List[str] = [
+            "Provide more details about your question."]
+
+    @model_validator(mode="after")
+    def validate_clarificaiton_questions(self):
+        if len(self.clarification_questions) < 1:
+            raise ValueError("At least one clarification question is required.")
+        return self
 
 
 class AuthConfig(BaseModel):

multi_llm_chatbot_backend/app/core/canvas_analysis.py

@@ -1,13 +1,10 @@
 import re
 import json
 import logging
-from typing import Dict, List, Tuple, Set
-from datetime import datetime
+from typing import Dict, List
 from collections import defaultdict
 
-from app.models.phd_canvas import CanvasInsight, CanvasSection
-from app.llm.improved_gemini_client import ImprovedGeminiClient
-from app.llm.improved_ollama_client import ImprovedOllamaClient
+from app.models.phd_canvas import CanvasInsight
 from app.config import get_settings
 
 logger = logging.getLogger(__name__)
@@ -123,11 +120,10 @@ class CanvasAnalysisService:
     
     async def _extract_insights_from_content(self, content: str, persona_id: str, message_id: str, chat_session_id: str) -> List[CanvasInsight]:
         """Extract actionable insights from content using LLM analysis"""
-        if not content or len(content.strip()) < 30:  # Lowered minimum length
+        if not content or len(content.strip()) < 30:
             return []
         
         try:
-            # Use LLM to extract key insights
             app_title = get_settings().app.title
             extraction_prompt = f"""
             Extract actionable insights from this {app_title} advisor response that would be valuable for a user's progress summary:
@@ -145,7 +141,7 @@ class CanvasAnalysisService:
             
             Return ONLY the JSON array, no other text.
             """
-            
+
             if self.llm_client:
                 try:
                     llm_response = await self.llm_client.generate(
@@ -154,7 +150,7 @@ class CanvasAnalysisService:
                         temperature=0.3,
                         max_tokens=500
                     )
-                    
+
                     # Clean the response to extract just the JSON
                     llm_response = llm_response.strip()
                     if llm_response.startswith('```json'):
@@ -162,10 +158,10 @@ class CanvasAnalysisService:
                     if llm_response.endswith('```'):
                         llm_response = llm_response[:-3]
                     llm_response = llm_response.strip()
-                    
+
                     insights_data = json.loads(llm_response)
                     insights = []
-                    
+
                     for item in insights_data:
                         if isinstance(item, dict) and "insight" in item:
                             insight = CanvasInsight(
@@ -178,18 +174,14 @@ class CanvasAnalysisService:
                             )
                             insights.append(insight)
                     
-                    logger.debug(f"LLM extracted {len(insights)} insights from {persona_id}")
-                    return insights
+                    if insights:
+                        logger.debug(f"LLM extracted {len(insights)} insights from {persona_id}")
+                        return insights
+                    
+                    logger.warning(f"LLM returned no parseable insights for {persona_id}, falling back to rule-based")
                     
-                except json.JSONDecodeError as je:
-                    logger.warning(f"Failed to parse LLM insights response for {persona_id}: {je}")
-                    logger.debug(f"Raw LLM response: {llm_response}")
-                    # Fallback to rule-based extraction
-                    return self._extract_insights_rule_based(content, persona_id, message_id, chat_session_id)
                 except Exception as llm_error:
                     logger.warning(f"LLM extraction failed for {persona_id}: {llm_error}")
-                    # Fallback to rule-based extraction
-                    return self._extract_insights_rule_based(content, persona_id, message_id, chat_session_id)
             
             # Fallback if no LLM available
             return self._extract_insights_rule_based(content, persona_id, message_id, chat_session_id)
@@ -216,19 +208,18 @@ class CanvasAnalysisService:
             if len(sentence) < 15:  # Lowered minimum length
                 continue
                 
-            # Check if sentence contains actionable language
-            is_actionable = any(re.search(pattern, sentence, re.IGNORECASE) for pattern in actionable_patterns)
+            match_count = sum(1 for pattern in actionable_patterns if re.search(pattern, sentence, re.IGNORECASE))
             
-            if is_actionable:
-                # Extract keywords
+            if match_count > 0:
                 keywords = self._extract_keywords_from_sentence(sentence)
+                confidence = round(0.55 + min(match_count * 0.05, 0.15), 2)
                 
                 insight = CanvasInsight(
                     content=sentence,
                     source_persona=persona_id,
                     source_message_id=message_id,
                     source_chat_session=chat_session_id,
-                    confidence_score=0.6,  # Lower confidence for rule-based
+                    confidence_score=confidence,
                     keywords=keywords
                 )
                 insights.append(insight)
@@ -300,4 +291,4 @@ class CanvasAnalysisService:
                 seen_content.add(content_key)
                 unique_insights.append(insight)
         
-        return unique_insights
+        return unique_insights

multi_llm_chatbot_backend/app/core/improved_orchestrator.py

@@ -3,8 +3,6 @@ from app.models.persona import Persona
 from app.core.session_manager import ConversationContext, get_session_manager
 from app.core.context_manager import get_context_manager
 from app.core.rag_manager import get_rag_manager
-from app.llm.llm_client import LLMClient
-from app.models.default_personas import is_valid_persona_id
 from app.config import get_settings
 
 import json
@@ -149,6 +147,7 @@ class ImprovedChatOrchestrator:
         # If this is not the first message, probably don't need clarification
         user_messages = [msg for msg in session.messages if msg.get('role') == 'user']
         if len(user_messages) > 1:
+            logger.info(f"Skipping clarification: session already has {len(user_messages)} user message(s)")
             return False
 
         # Check for vague patterns - FIXED to handle "I am" vs "I'm"
@@ -165,47 +164,94 @@ class ImprovedChatOrchestrator:
         ]
 
         orch_cfg = get_settings().orchestrator
-        
         user_lower = user_input.lower().strip()
-        
-        logger.info(f"Checking clarification for: '{user_input}' (lowercase: '{user_lower}')")
-        
-        for pattern in vague_patterns:
-            if re.search(pattern, user_lower):
-                logger.info(f"CLARIFICATION TRIGGERED: Pattern '{pattern}' matched input '{user_input}'")
-                return True
-        
-        # Check if input is too short and vague
         word_count = len(user_input.split())
+
+        logger.info(f"Checking clarification for: {user_input} ({word_count} words)")
+
         has_specific_keywords = any(
             keyword in user_lower for keyword in orch_cfg.specific_keywords
         )
-        
-        if word_count < orch_cfg.min_words_without_keywords and not has_specific_keywords:
-            logger.info(f"CLARIFICATION TRIGGERED: Short input ({word_count} words) without specific keywords")
-            return True
-        
-        logger.info(f"NO CLARIFICATION: Input has {word_count} words, specific keywords: {has_specific_keywords}")
-        return False
+        if has_specific_keywords:
+            logger.info("NO CLARIFICATION: input contains specific keywords")
+            return False
+
+        if word_count >= orch_cfg.min_words_without_keywords:
+            logger.info(
+                    f"NO CLARIFICATION: input has {word_count} words "
+                    f"(>= {orch_cfg.min_words_without_keywords} threshold)")
+            return False
+
+        for pattern in vague_patterns:
+            if re.search(pattern, user_lower):
+                logger.info(f"CLARIFICATION TRIGGERED: pattern `{pattern}` matched `{user_input}`")
+                return True
+
+        logger.info("CLARIFICATION TRIGGERED: short input (%d words) without specific keywords", word_count)
+        return True
     
-    async def _generate_clarification_question(self, session: ConversationContext) -> str:
+    async def generate_contextual_clarification(self, user_input: str) -> Dict[str, Any]:
         """
-        Generate a clarification question based on the conversation context.
-        Questions are driven by config.yaml → orchestrator.clarification_questions.
+        Use the LLM to produce a clarification question and clickable
+        suggestions that are tailored to what the user actually typed.
+        Falls back to the static values in config.yaml if the LLM call fails.
         """
         orch_cfg = get_settings().orchestrator
-        questions = orch_cfg.clarification_questions or [
-            "Could you provide more details about what you need help with?"
-        ]
-        # Return the first option for now (could be made smarter with AI)
-        return questions[0]
-    
-    def _get_clarification_suggestions(self) -> List[str]:
-        """Get suggestions for clarification from config."""
-        orch_cfg = get_settings().orchestrator
-        return orch_cfg.clarification_suggestions or [
-            "Provide more details about your question"
-        ]
+
+        advisor_list = ", ".join(
+            f"{p.name} ({p.id})" for p in self.personas.values()
+        )
+
+        system_prompt = (
+            "You are a helpful routing assistant. The user's message is too "
+            "vague to send to the advisors. Produce a short clarifying question "
+            "and exactly 4 clickable suggestion buttons the user could press.\n\n"
+            "Reply ONLY with valid JSON — no markdown, no extra text:\n"
+            '{"question": "...", "suggestions": ["...", "...", "...", "..."]}\n\n'
+            "Keep the question to one sentence. Each suggestion should be a "
+            "complete sentence the user could send as their next message."
+        )
+
+        user_prompt = (
+            f"User said: \"{user_input}\"\n"
+            f"Available advisors: {advisor_list}\n\n"
+            "Generate a clarifying question and 4 suggestion buttons that "
+            "relate to what the user said and steer toward the advisors above."
+        )
+
+        try:
+            llm = next(iter(self.personas.values())).llm
+            raw = await llm.generate(
+                system_prompt=system_prompt,
+                context=[{"role": "user", "content": user_prompt}],
+                temperature=0.4,
+                max_tokens=1024,
+            )
+
+            cleaned = raw.strip()
+            cleaned = re.sub(r"```(?:json)?", "", cleaned).strip()
+
+            json_match = re.search(r"\{.*\}", cleaned, re.DOTALL)
+            if json_match:
+                cleaned = json_match.group(0)
+
+            parsed = json.loads(cleaned)
+            question = parsed.get("question", "").strip()
+            suggestions = parsed.get("suggestions", [])
+
+            if question and isinstance(suggestions, list) and len(suggestions) >= 2:
+                logger.info(f"LLM clarification generated for: {user_input}")
+                return {"question": question, "suggestions": suggestions[:4]}
+
+        except Exception as e:
+            logger.error(f"LLM clarification failed, using config fallback: {e}")
+
+        fallback_questions = orch_cfg.clarification_questions
+        fallback_suggestions = orch_cfg.clarification_suggestions
+        return {
+            "question": fallback_questions[0],
+            "suggestions": fallback_suggestions,
+        }
     
     async def _generate_persona_responses(self, session: ConversationContext, response_length: str = "medium"):
         """

multi_llm_chatbot_backend/app/llm/improved_gemini_client.py

@@ -1,4 +1,6 @@
 import httpx
+import os
+import re
 from typing import List
 from app.llm.llm_client import LLMClient
 from app.core.context_manager import get_context_manager
@@ -91,7 +93,12 @@ class ImprovedGeminiClient(LLMClient):
                     logger.error(f"Invalid candidate structure: {candidate}")
                     return "I apologize, but I received an unexpected response format. Please try again."
                 
-                text = candidate["content"]["parts"][0].get("text", "").strip()
+                parts = candidate["content"]["parts"]
+                text = "\n\n".join(
+                    p.get("text", "")
+                    for p in parts
+                    if not p.get("thought") and p.get("text", "").strip()
+                ).strip()
                 
                 if not text:
                     logger.warning("Empty response from Gemini")
@@ -110,21 +117,9 @@ class ImprovedGeminiClient(LLMClient):
             return "I encountered an unexpected error. Please try again."
     
     def _clean_response(self, response: str) -> str:
-        """Clean up response text"""
-        # Remove common issues
-        """response = response.strip()
-        
-        # Remove duplicate spaces and normalize
-        response = ' '.join(response.split())
-        
-        return response"""
-        response = response.strip()
-
-        # Preserve line breaks for Markdown; only trim right-side spaces and collapse 3+ blank lines
+        """Clean up response text, preserving Markdown formatting."""
         response = response.replace("\r\n", "\n").replace("\r", "\n")
         lines = [ln.rstrip() for ln in response.split("\n")]
-
-        import re
         response = re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()
 
         return response

multi_llm_chatbot_backend/app/llm/improved_ollama_client.py

@@ -94,11 +94,6 @@ class ImprovedOllamaClient(LLMClient):
         for pattern in fluff_patterns:
             response = response.replace(pattern, "").strip()
         
-        """# Normalize whitespace
-        response = ' '.join(response.split())
-        
-        return response"""
-        # Preserve Markdown line breaks; trim right-side spaces; collapse very long blank runs
         response = response.replace("\r\n", "\n").replace("\r", "\n")
         lines = [ln.rstrip() for ln in response.split("\n")]