add canvas backend

multi_llm_chatbot_backend/app/api/routes/phd_canvas.py

@@ -0,0 +1,288 @@
+from fastapi import APIRouter, HTTPException, Depends, status, BackgroundTasks
+from typing import Dict, Optional
+from datetime import datetime
+import logging
+
+from app.models.user import User
+from app.models.phd_canvas import PhdCanvas, CanvasResponse, UpdateCanvasRequest
+from app.core.auth import get_current_active_user
+
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+# Request/Response models
+class CanvasStatsResponse(BaseModel):
+    total_insights: int
+    total_sections: int
+    last_updated: Optional[str] = None
+    last_chat_processed: Optional[str] = None
+    created_at: Optional[str] = None
+    auto_update: bool = True
+    sections_breakdown: Dict = {}
+
+class PrintCanvasResponse(BaseModel):
+    user_id: str
+    generated_at: str
+    total_insights: int
+    last_updated: Optional[str] = None
+    sections: list
+    metadata: Dict
+
+def get_canvas_manager():
+    """Lazy import to avoid circular dependency"""
+    from app.core.canvas_manager import get_canvas_manager
+    return get_canvas_manager()
+
+@router.get("/phd-canvas", response_model=CanvasResponse)
+async def get_phd_canvas(
+    current_user: User = Depends(get_current_active_user)
+):
+    """Get the user's PhD Canvas, creating it if it doesn't exist"""
+    try:
+        canvas_manager = get_canvas_manager()
+        canvas = await canvas_manager.get_or_create_canvas(str(current_user.id))
+        
+        # Convert to response model
+        return CanvasResponse(
+            id=str(canvas.id),
+            user_id=str(canvas.user_id),
+            sections=canvas.sections,
+            created_at=canvas.created_at,
+            last_updated=canvas.last_updated,
+            last_chat_processed=canvas.last_chat_processed,
+            total_insights=canvas.total_insights,
+            auto_update=canvas.auto_update,
+            print_optimized=canvas.print_optimized
+        )
+        
+    except Exception as e:
+        logger.error(f"Error getting PhD canvas for user {current_user.id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to retrieve PhD canvas"
+        )
+
+@router.post("/phd-canvas/update", response_model=CanvasResponse)
+async def update_phd_canvas(
+    request: UpdateCanvasRequest,
+    background_tasks: BackgroundTasks,
+    current_user: User = Depends(get_current_active_user)
+):
+    """Update the PhD Canvas with latest insights from chat sessions"""
+    try:
+        canvas_manager = get_canvas_manager()
+        
+        # For large updates, run in background
+        if request.force_full_update:
+            logger.info(f"Queuing background canvas update for user {current_user.id}")
+            background_tasks.add_task(
+                _background_canvas_update,
+                str(current_user.id),
+                request
+            )
+            
+            # Return current canvas immediately
+            canvas = await canvas_manager.get_or_create_canvas(str(current_user.id))
+        else:
+            # For incremental updates, process immediately
+            logger.info(f"Processing incremental canvas update for user {current_user.id}")
+            canvas = await canvas_manager.update_canvas(str(current_user.id), request)
+        
+        return CanvasResponse(
+            id=str(canvas.id),
+            user_id=str(canvas.user_id),
+            sections=canvas.sections,
+            created_at=canvas.created_at,
+            last_updated=canvas.last_updated,
+            last_chat_processed=canvas.last_chat_processed,
+            total_insights=canvas.total_insights,
+            auto_update=canvas.auto_update,
+            print_optimized=canvas.print_optimized
+        )
+        
+    except Exception as e:
+        logger.error(f"Error updating PhD canvas for user {current_user.id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to update PhD canvas"
+        )
+
+@router.get("/phd-canvas/stats", response_model=CanvasStatsResponse)
+async def get_canvas_stats(
+    current_user: User = Depends(get_current_active_user)
+):
+    """Get statistics about the user's PhD Canvas"""
+    try:
+        canvas_manager = get_canvas_manager()
+        stats = await canvas_manager.get_canvas_stats(str(current_user.id))
+        
+        return CanvasStatsResponse(
+            total_insights=stats.get("total_insights", 0),
+            total_sections=stats.get("total_sections", 0),
+            last_updated=stats.get("last_updated").isoformat() if stats.get("last_updated") else None,
+            last_chat_processed=stats.get("last_chat_processed").isoformat() if stats.get("last_chat_processed") else None,
+            created_at=stats.get("created_at").isoformat() if stats.get("created_at") else None,
+            auto_update=stats.get("auto_update", True),
+            sections_breakdown=stats.get("sections_breakdown", {})
+        )
+        
+    except Exception as e:
+        logger.error(f"Error getting canvas stats for user {current_user.id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to retrieve canvas statistics"
+        )
+
+@router.get("/phd-canvas/print", response_model=PrintCanvasResponse)
+async def get_print_canvas(
+    current_user: User = Depends(get_current_active_user)
+):
+    """Get PhD Canvas formatted for printing"""
+    try:
+        canvas_manager = get_canvas_manager()
+        print_data = await canvas_manager.export_canvas_for_printing(str(current_user.id))
+        
+        return PrintCanvasResponse(
+            user_id=print_data["user_id"],
+            generated_at=print_data["generated_at"].isoformat(),
+            total_insights=print_data["total_insights"],
+            last_updated=print_data["last_updated"].isoformat() if print_data["last_updated"] else None,
+            sections=print_data["sections"],
+            metadata=print_data["metadata"]
+        )
+        
+    except Exception as e:
+        logger.error(f"Error exporting print canvas for user {current_user.id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to export canvas for printing"
+        )
+
+@router.post("/phd-canvas/auto-update")
+async def trigger_auto_update(
+    background_tasks: BackgroundTasks,
+    current_user: User = Depends(get_current_active_user)
+):
+    """Trigger automatic canvas update (typically called when user opens canvas page)"""
+    try:
+        canvas_manager = get_canvas_manager()
+        
+        # Get current canvas to check if auto-update is needed
+        canvas = await canvas_manager.get_or_create_canvas(str(current_user.id))
+        
+        if not canvas.auto_update:
+            return {"message": "Auto-update is disabled for this canvas"}
+        
+        # Check if we need to update (has been more than 1 hour since last update)
+        from datetime import timedelta
+        needs_update = (
+            canvas.last_updated is None or 
+            (datetime.utcnow() - canvas.last_updated) > timedelta(hours=1)
+        )
+        
+        if needs_update:
+            logger.info(f"Auto-updating canvas for user {current_user.id}")
+            
+            # Queue incremental update in background
+            background_tasks.add_task(
+                _background_canvas_update,
+                str(current_user.id),
+                UpdateCanvasRequest(force_full_update=False)
+            )
+            
+            return {"message": "Canvas update queued", "status": "updating"}
+        else:
+            return {"message": "Canvas is up to date", "status": "current"}
+            
+    except Exception as e:
+        logger.error(f"Error triggering auto-update for user {current_user.id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to trigger canvas auto-update"
+        )
+
+@router.delete("/phd-canvas")
+async def delete_canvas(
+    current_user: User = Depends(get_current_active_user)
+):
+    """Delete the user's PhD Canvas"""
+    try:
+        canvas_manager = get_canvas_manager()
+        deleted = await canvas_manager.delete_canvas(str(current_user.id))
+        
+        if deleted:
+            return {"message": "PhD Canvas deleted successfully"}
+        else:
+            return {"message": "No canvas found to delete"}
+            
+    except Exception as e:
+        logger.error(f"Error deleting canvas for user {current_user.id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to delete PhD canvas"
+        )
+
+@router.get("/phd-canvas/refresh")
+async def refresh_canvas_data(
+    background_tasks: BackgroundTasks,
+    current_user: User = Depends(get_current_active_user)
+):
+    """Force refresh canvas with full update from all chats"""
+    try:
+        logger.info(f"Force refreshing canvas for user {current_user.id}")
+        
+        # Queue full update in background
+        background_tasks.add_task(
+            _background_canvas_update,
+            str(current_user.id),
+            UpdateCanvasRequest(force_full_update=True)
+        )
+        
+        return {
+            "message": "Full canvas refresh initiated",
+            "status": "processing",
+            "estimated_completion": "2-3 minutes"
+        }
+        
+    except Exception as e:
+        logger.error(f"Error refreshing canvas for user {current_user.id}: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to refresh PhD canvas"
+        )
+
+# Background task functions
+async def _background_canvas_update(user_id: str, request: UpdateCanvasRequest):
+    """Background task to update canvas without blocking the API response"""
+    try:
+        logger.info(f"Starting background canvas update for user {user_id}")
+        canvas_manager = get_canvas_manager()
+        
+        updated_canvas = await canvas_manager.update_canvas(user_id, request)
+        
+        logger.info(f"Background canvas update completed for user {user_id}. Total insights: {updated_canvas.total_insights}")
+        
+    except Exception as e:
+        logger.error(f"Error in background canvas update for user {user_id}: {e}")
+        import traceback
+        logger.error(f"Full traceback: {traceback.format_exc()}")
+
+# Health check endpoint
+@router.get("/phd-canvas/health")
+async def canvas_health_check():
+    """Health check for canvas service"""
+    try:
+        canvas_manager = get_canvas_manager()
+        return {
+            "status": "healthy",
+            "service": "phd-canvas",
+            "timestamp": datetime.utcnow().isoformat()
+        }
+    except Exception as e:
+        logger.error(f"Canvas health check failed: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Canvas service is not healthy"
+        )

multi_llm_chatbot_backend/app/core/canvas_analysis.py

@@ -0,0 +1,382 @@
+import re
+import json
+import logging
+from typing import Dict, List, Tuple, Set
+from datetime import datetime
+from collections import defaultdict
+
+from app.models.phd_canvas import CanvasInsight, CanvasSection
+from app.llm.improved_gemini_client import ImprovedGeminiClient
+from app.llm.improved_ollama_client import ImprovedOllamaClient
+
+logger = logging.getLogger(__name__)
+
+class CanvasAnalysisService:
+    """Service for extracting and categorizing insights from chat messages"""
+    
+    def __init__(self, llm_client=None):
+        self.llm_client = llm_client
+        
+        # Predefined section mappings for semantic analysis
+        self.section_keywords = {
+            "research_progress": [
+                "progress", "milestone", "completed", "finished", "accomplished", 
+                "achieved", "timeline", "deadline", "chapter", "draft", "version"
+            ],
+            "methodology": [
+                "method", "methodology", "approach", "design", "data collection",
+                "survey", "interview", "experiment", "analysis", "statistical", 
+                "qualitative", "quantitative", "mixed methods"
+            ],
+            "theoretical_framework": [
+                "theory", "theoretical", "framework", "concept", "model",
+                "literature", "author", "philosophy", "paradigm", "assumption"
+            ],
+            "challenges_obstacles": [
+                "challenge", "problem", "difficulty", "obstacle", "stuck",
+                "confused", "frustrated", "struggle", "barrier", "issue"
+            ],
+            "next_steps": [
+                "next", "plan", "should", "will", "going to", "need to",
+                "action", "step", "priority", "focus", "goal", "objective"
+            ],
+            "writing_communication": [
+                "write", "writing", "paper", "thesis", "dissertation", "publication",
+                "communication", "presentation", "defense", "draft", "revision"
+            ],
+            "career_development": [
+                "career", "job", "position", "networking", "conference",
+                "skills", "CV", "resume", "application", "fellowship", "grant"
+            ],
+            "literature_review": [
+                "literature", "sources", "papers", "articles", "bibliography",
+                "citation", "author", "study", "research", "gap", "review"
+            ],
+            "data_analysis": [
+                "data", "analysis", "results", "findings", "statistics",
+                "coding", "software", "tool", "visualization", "pattern"
+            ],
+            "motivation_mindset": [
+                "motivation", "confidence", "stress", "anxiety", "balance",
+                "mental health", "mindset", "support", "overwhelmed", "burnout"
+            ]
+        }
+    
+    async def extract_insights_from_messages(self, messages: List[Dict], chat_session_id: str) -> List[CanvasInsight]:
+        """Extract actionable insights from chat messages using semantic analysis"""
+        try:
+            insights = []
+            
+            for message in messages:
+                if message.get("type") == "assistant" and "responses" in message:
+                    # Handle multi-persona responses
+                    for response in message.get("responses", []):
+                        persona_insights = await self._extract_insights_from_persona_response(
+                            response, message.get("id"), chat_session_id
+                        )
+                        insights.extend(persona_insights)
+                elif message.get("role") == "assistant" and message.get("content"):
+                    # Handle single response format
+                    persona_insights = await self._extract_insights_from_content(
+                        message.get("content", ""), "assistant", 
+                        message.get("id"), chat_session_id
+                    )
+                    insights.extend(persona_insights)
+            
+            # Remove duplicates and low-confidence insights
+            unique_insights = self._deduplicate_insights(insights)
+            high_confidence_insights = [i for i in unique_insights if i.confidence_score >= 0.6]
+            
+            logger.info(f"Extracted {len(high_confidence_insights)} insights from {len(messages)} messages")
+            return high_confidence_insights
+            
+        except Exception as e:
+            logger.error(f"Error extracting insights from messages: {e}")
+            return []
+    
+    async def _extract_insights_from_persona_response(self, response: Dict, message_id: str, chat_session_id: str) -> List[CanvasInsight]:
+        """Extract insights from a single persona response"""
+        persona_id = response.get("persona_id", "unknown")
+        content = response.get("content", "")
+        
+        return await self._extract_insights_from_content(content, persona_id, message_id, chat_session_id)
+    
+    async def _extract_insights_from_content(self, content: str, persona_id: str, message_id: str, chat_session_id: str) -> List[CanvasInsight]:
+        """Extract actionable insights from content using LLM analysis"""
+        if not content or len(content.strip()) < 50:
+            return []
+        
+        try:
+            # Use LLM to extract key insights
+            extraction_prompt = f"""
+            Extract actionable insights from this PhD advisor response that would be valuable for a student's progress summary:
+
+            PERSONA: {persona_id}
+            CONTENT: {content}
+
+            Return a JSON list of insights. Each insight should be:
+            - Actionable and specific to PhD progress
+            - 1-2 sentences long
+            - Valuable for advisor meetings
+            - Not generic advice
+
+            Format: [{{"insight": "specific actionable insight here", "keywords": ["keyword1", "keyword2"]}}]
+            
+            Return ONLY the JSON array, no other text.
+            """
+            
+            if self.llm_client:
+                llm_response = await self.llm_client.generate(
+                    system_prompt="You are an expert at extracting actionable PhD guidance from advisor responses.",
+                    context=[{"role": "user", "content": extraction_prompt}],
+                    temperature=0.3,
+                    max_tokens=500
+                )
+                
+                try:
+                    insights_data = json.loads(llm_response.strip())
+                    insights = []
+                    
+                    for item in insights_data:
+                        if isinstance(item, dict) and "insight" in item:
+                            insight = CanvasInsight(
+                                content=item["insight"],
+                                source_persona=persona_id,
+                                source_message_id=message_id,
+                                source_chat_session=chat_session_id,
+                                confidence_score=0.8,  # High confidence from LLM extraction
+                                keywords=item.get("keywords", [])
+                            )
+                            insights.append(insight)
+                    
+                    return insights
+                    
+                except json.JSONDecodeError:
+                    logger.warning(f"Failed to parse LLM insights response for {persona_id}")
+                    # Fallback to rule-based extraction
+                    return self._extract_insights_rule_based(content, persona_id, message_id, chat_session_id)
+            
+            # Fallback if no LLM available
+            return self._extract_insights_rule_based(content, persona_id, message_id, chat_session_id)
+            
+        except Exception as e:
+            logger.error(f"Error extracting insights from {persona_id} content: {e}")
+            return []
+    
+    def _extract_insights_rule_based(self, content: str, persona_id: str, message_id: str, chat_session_id: str) -> List[CanvasInsight]:
+        """Fallback rule-based insight extraction"""
+        insights = []
+        
+        # Extract actionable sentences
+        sentences = re.split(r'[.!?]+', content)
+        actionable_patterns = [
+            r"(?:you should|consider|try|focus on|prioritize|next step|recommended?|suggest)",
+            r"(?:action item|goal|objective|plan|strategy)",
+            r"(?:deadline|timeline|schedule|by \w+)",
+            r"(?:complete|finish|work on|tackle|address)"
+        ]
+        
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if len(sentence) < 20:  # Skip very short sentences
+                continue
+                
+            # Check if sentence contains actionable language
+            is_actionable = any(re.search(pattern, sentence, re.IGNORECASE) for pattern in actionable_patterns)
+            
+            if is_actionable:
+                # Extract keywords
+                keywords = self._extract_keywords_from_sentence(sentence)
+                
+                insight = CanvasInsight(
+                    content=sentence,
+                    source_persona=persona_id,
+                    source_message_id=message_id,
+                    source_chat_session=chat_session_id,
+                    confidence_score=0.6,  # Lower confidence for rule-based
+                    keywords=keywords
+                )
+                insights.append(insight)
+        
+        return insights[:3]  # Limit to 3 insights per message to avoid noise
+    
+    def _extract_keywords_from_sentence(self, sentence: str) -> List[str]:
+        """Extract relevant keywords from a sentence"""
+        # Simple keyword extraction
+        words = re.findall(r'\b\w{4,}\b', sentence.lower())  # Words with 4+ chars
+        
+        # Filter out common words
+        stop_words = {
+            "should", "could", "would", "your", "research", "work", "study", 
+            "consider", "think", "important", "also", "really", "maybe",
+            "probably", "definitely", "certainly", "particular", "specific"
+        }
+        
+        keywords = [word for word in words if word not in stop_words]
+        return keywords[:5]  # Limit to 5 keywords
+    
+    def categorize_insights(self, insights: List[CanvasInsight]) -> Dict[str, List[CanvasInsight]]:
+        """Categorize insights into canvas sections using semantic analysis"""
+        categorized = defaultdict(list)
+        
+        for insight in insights:
+            section = self._determine_section(insight)
+            categorized[section].append(insight)
+        
+        return dict(categorized)
+    
+    def _determine_section(self, insight: CanvasInsight) -> str:
+        """Determine which canvas section an insight belongs to"""
+        content_lower = insight.content.lower()
+        keywords_lower = [kw.lower() for kw in insight.keywords]
+        all_text = content_lower + " " + " ".join(keywords_lower)
+        
+        # Score each section based on keyword matches
+        section_scores = {}
+        
+        for section_key, section_keywords in self.section_keywords.items():
+            score = 0
+            for keyword in section_keywords:
+                if keyword in all_text:
+                    score += 1
+            
+            # Bonus for persona-specific insights
+            persona_bonuses = {
+                "methodologist": {"methodology": 2, "data_analysis": 1},
+                "theorist": {"theoretical_framework": 2, "literature_review": 1},
+                "pragmatist": {"next_steps": 2, "research_progress": 1},
+                "socratic": {"theoretical_framework": 1, "challenges_obstacles": 1},
+                "motivator": {"motivation_mindset": 2, "career_development": 1},
+                "critic": {"challenges_obstacles": 1, "writing_communication": 1},
+                "empathetic": {"motivation_mindset": 2},
+                "visionary": {"career_development": 1, "research_progress": 1}
+            }
+            
+            if insight.source_persona in persona_bonuses:
+                score += persona_bonuses[insight.source_persona].get(section_key, 0)
+            
+            section_scores[section_key] = score
+        
+        # Return section with highest score, default to general category
+        best_section = max(section_scores, key=section_scores.get) if section_scores else "research_progress"
+        
+        # If no keywords matched, categorize by persona type
+        if section_scores[best_section] == 0:
+            persona_defaults = {
+                "methodologist": "methodology",
+                "theorist": "theoretical_framework", 
+                "pragmatist": "next_steps",
+                "socratic": "theoretical_framework",
+                "motivator": "motivation_mindset",
+                "critic": "challenges_obstacles",
+                "empathetic": "motivation_mindset",
+                "visionary": "career_development",
+                "storyteller": "writing_communication",
+                "minimalist": "next_steps"
+            }
+            best_section = persona_defaults.get(insight.source_persona, "research_progress")
+        
+        return best_section
+    
+    def _deduplicate_insights(self, insights: List[CanvasInsight]) -> List[CanvasInsight]:
+        """Remove duplicate insights based on semantic similarity"""
+        if not insights:
+            return []
+        
+        unique_insights = []
+        seen_contents = set()
+        
+        for insight in insights:
+            # Simple deduplication based on content similarity
+            content_normalized = re.sub(r'\s+', ' ', insight.content.lower().strip())
+            
+            # Check for substantial overlap with existing insights
+            is_duplicate = False
+            for seen_content in seen_contents:
+                if self._calculate_similarity(content_normalized, seen_content) > 0.8:
+                    is_duplicate = True
+                    break
+            
+            if not is_duplicate:
+                unique_insights.append(insight)
+                seen_contents.add(content_normalized)
+        
+        return unique_insights
+    
+    def _calculate_similarity(self, text1: str, text2: str) -> float:
+        """Calculate simple similarity between two texts"""
+        words1 = set(text1.split())
+        words2 = set(text2.split())
+        
+        if not words1 or not words2:
+            return 0.0
+        
+        intersection = words1.intersection(words2)
+        union = words1.union(words2)
+        
+        return len(intersection) / len(union) if union else 0.0
+
+    def prioritize_insights(self, insights: List[CanvasInsight]) -> List[CanvasInsight]:
+        """Prioritize insights based on relevance and actionability"""
+        
+        def calculate_priority_score(insight: CanvasInsight) -> float:
+            score = insight.confidence_score
+            
+            # Boost score for actionable language
+            actionable_terms = ["should", "need to", "must", "plan", "goal", "deadline", "complete"]
+            content_lower = insight.content.lower()
+            for term in actionable_terms:
+                if term in content_lower:
+                    score += 0.1
+            
+            # Boost score for specific vs generic advice
+            specific_terms = ["chapter", "data", "analysis", "method", "theory", "timeline"]
+            for term in specific_terms:
+                if term in content_lower:
+                    score += 0.05
+            
+            # Boost score for certain personas
+            persona_boosts = {
+                "pragmatist": 0.1,    # Action-oriented insights are valuable
+                "methodologist": 0.08, # Methodological insights are important
+                "critic": 0.06        # Critical feedback is valuable
+            }
+            score += persona_boosts.get(insight.source_persona, 0)
+            
+            return min(score, 1.0)  # Cap at 1.0
+        
+        # Calculate scores and sort
+        for insight in insights:
+            insight.confidence_score = calculate_priority_score(insight)
+        
+        return sorted(insights, key=lambda x: x.confidence_score, reverse=True)
+
+    async def generate_section_summary(self, section: CanvasSection) -> str:
+        """Generate a summary for a canvas section using LLM"""
+        if not section.insights or not self.llm_client:
+            return f"Key insights for {section.title}"
+        
+        try:
+            insights_text = "\n".join([f"- {insight.content}" for insight in section.insights])
+            
+            prompt = f"""
+            Create a 1-2 sentence summary of these PhD insights for the section "{section.title}":
+            
+            {insights_text}
+            
+            The summary should be concise, professional, and suitable for sharing with an advisor.
+            Return only the summary text, no other content.
+            """
+            
+            summary = await self.llm_client.generate(
+                system_prompt="You create concise summaries of academic insights.",
+                context=[{"role": "user", "content": prompt}],
+                temperature=0.2,
+                max_tokens=100
+            )
+            
+            return summary.strip()
+            
+        except Exception as e:
+            logger.error(f"Error generating section summary: {e}")
+            return f"Key insights and guidance for {section.title}"

multi_llm_chatbot_backend/app/core/canvas_database.py

@@ -0,0 +1,76 @@
+import logging
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+logger = logging.getLogger(__name__)
+
+async def setup_canvas_collections(db: AsyncIOMotorDatabase):
+    """Setup MongoDB collections and indexes for PhD Canvas"""
+    try:
+        # Create indexes for phd_canvases collection
+        canvas_indexes = [
+            # User lookup index
+            ("user_id", 1),
+            # Query optimization indexes  
+            [("user_id", 1), ("last_updated", -1)],
+            [("user_id", 1), ("created_at", -1)],
+            ("last_updated", -1),  # For cleanup operations
+        ]
+        
+        collection = db.phd_canvases
+        
+        # Create indexes
+        for index in canvas_indexes:
+            if isinstance(index, list):
+                # Compound index
+                await collection.create_index(index)
+                logger.info(f"Created compound index: {index}")
+            else:
+                # Simple index
+                await collection.create_index(index)
+                logger.info(f"Created index: {index}")
+        
+        # Ensure TTL index for old canvases (optional cleanup after 2 years)
+        await collection.create_index(
+            "created_at", 
+            expireAfterSeconds=63072000  # 2 years in seconds
+        )
+        logger.info("Created TTL index for canvas cleanup")
+        
+        logger.info("PhD Canvas database setup completed successfully")
+        
+    except Exception as e:
+        logger.error(f"Error setting up canvas collections: {e}")
+        raise
+
+async def cleanup_old_canvas_data(db: AsyncIOMotorDatabase):
+    """Cleanup old or orphaned canvas data (maintenance function)"""
+    try:
+        # Find canvases with no corresponding users (orphaned data)
+        pipeline = [
+            {
+                "$lookup": {
+                    "from": "users",
+                    "localField": "user_id", 
+                    "foreignField": "_id",
+                    "as": "user_data"
+                }
+            },
+            {
+                "$match": {
+                    "user_data": {"$size": 0}  # No matching user found
+                }
+            }
+        ]
+        
+        orphaned_canvases = await db.phd_canvases.aggregate(pipeline).to_list(length=100)
+        
+        if orphaned_canvases:
+            orphaned_ids = [canvas["_id"] for canvas in orphaned_canvases]
+            result = await db.phd_canvases.delete_many({"_id": {"$in": orphaned_ids}})
+            logger.info(f"Cleaned up {result.deleted_count} orphaned canvas records")
+        else:
+            logger.info("No orphaned canvas records found")
+            
+    except Exception as e:
+        logger.error(f"Error during canvas cleanup: {e}")
+

multi_llm_chatbot_backend/app/core/canvas_manager.py

@@ -0,0 +1,337 @@
+import logging
+from typing import Dict, List, Optional
+from datetime import datetime, timedelta
+from bson import ObjectId
+
+from app.models.phd_canvas import PhdCanvas, CanvasInsight, UpdateCanvasRequest
+from app.core.canvas_analysis import CanvasAnalysisService
+# Remove this import: from app.core.database import get_database
+from app.core.bootstrap import llm
+
+logger = logging.getLogger(__name__)
+
+class CanvasManager:
+    """Manages PhD Canvas creation, updates, and incremental processing"""
+    
+    def __init__(self):
+        self.analysis_service = CanvasAnalysisService(llm_client=llm)
+        self._db = None
+    
+    def get_database(self):
+        """Lazy database connection to avoid circular imports"""
+        if not self._db:
+            # Import here to avoid circular import
+            from app.core.database import get_database
+            self._db = get_database()
+        return self._db
+    
+    async def get_or_create_canvas(self, user_id: str) -> PhdCanvas:
+        """Get existing canvas or create new one for user"""
+        try:
+            db = self.get_database()
+            user_object_id = ObjectId(user_id)
+            
+            # Try to find existing canvas
+            canvas_doc = await db.phd_canvases.find_one({"user_id": user_object_id})
+            
+            if canvas_doc:
+                # Convert to PhdCanvas model
+                canvas = PhdCanvas(**canvas_doc)
+                logger.info(f"Found existing canvas for user {user_id} with {canvas.total_insights} insights")
+                return canvas
+            else:
+                # Create new canvas
+                canvas = PhdCanvas(user_id=user_object_id)
+                
+                # Insert into database
+                result = await db.phd_canvases.insert_one(canvas.dict(by_alias=True))
+                canvas.id = result.inserted_id
+                
+                logger.info(f"Created new canvas for user {user_id}")
+                return canvas
+                
+        except Exception as e:
+            logger.error(f"Error getting/creating canvas for user {user_id}: {e}")
+            # Return empty canvas as fallback
+            return PhdCanvas(user_id=ObjectId(user_id))
+    
+    async def update_canvas(self, user_id: str, request: UpdateCanvasRequest) -> PhdCanvas:
+        """Update canvas with latest insights from chat sessions"""
+        try:
+            db = self.get_database()
+            canvas = await self.get_or_create_canvas(user_id)
+            
+            logger.info(f"Updating canvas for user {user_id}, force_full={request.force_full_update}")
+            
+            # Determine which chats to process
+            if request.force_full_update:
+                # Process all chats
+                chat_sessions = await self._get_all_user_chat_sessions(user_id)
+                logger.info(f"Force full update: processing {len(chat_sessions)} total chat sessions")
+            else:
+                # Process only chats created/updated after last canvas update
+                chat_sessions = await self._get_new_chat_sessions(user_id, canvas.last_chat_processed)
+                logger.info(f"Incremental update: processing {len(chat_sessions)} new chat sessions since {canvas.last_chat_processed}")
+            
+            if not chat_sessions:
+                logger.info("No new chat sessions to process")
+                return canvas
+            
+            # Filter chat sessions if specific ones requested
+            if request.include_chat_sessions:
+                chat_sessions = [
+                    chat for chat in chat_sessions 
+                    if str(chat["_id"]) in request.include_chat_sessions
+                ]
+                logger.info(f"Filtered to {len(chat_sessions)} specifically requested chat sessions")
+            
+            # Process each chat session for insights
+            all_new_insights = []
+            processed_chat_ids = []
+            
+            for chat_session in chat_sessions:
+                try:
+                    chat_id = str(chat_session["_id"])
+                    messages = chat_session.get("messages", [])
+                    
+                    if not messages:
+                        continue
+                    
+                    logger.info(f"Processing chat {chat_id} with {len(messages)} messages")
+                    
+                    # Extract insights from this chat session
+                    session_insights = await self.analysis_service.extract_insights_from_messages(
+                        messages, chat_id
+                    )
+                    
+                    if session_insights:
+                        all_new_insights.extend(session_insights)
+                        processed_chat_ids.append(chat_id)
+                        logger.info(f"Extracted {len(session_insights)} insights from chat {chat_id}")
+                    
+                except Exception as e:
+                    logger.error(f"Error processing chat session {chat_session.get('_id')}: {e}")
+                    continue
+            
+            if all_new_insights:
+                # Categorize insights by section
+                categorized_insights = self.analysis_service.categorize_insights(all_new_insights)
+                
+                # Update canvas sections
+                sections_updated = 0
+                for section_key, insights in categorized_insights.items():
+                    if request.exclude_sections and section_key in request.exclude_sections:
+                        continue
+                    
+                    # Prioritize insights before adding
+                    prioritized_insights = self.analysis_service.prioritize_insights(insights)
+                    
+                    # Limit insights per section to avoid overwhelming canvas
+                    max_insights_per_section = 10
+                    limited_insights = prioritized_insights[:max_insights_per_section]
+                    
+                    if limited_insights:
+                        canvas.update_section(section_key, limited_insights)
+                        sections_updated += 1
+                        logger.info(f"Updated section '{section_key}' with {len(limited_insights)} insights")
+                
+                # Update canvas metadata
+                canvas.last_chat_processed = datetime.utcnow()
+                canvas.last_updated = datetime.utcnow()
+                
+                # Save updated canvas to database
+                await self._save_canvas(canvas)
+                
+                logger.info(f"Canvas update completed: {len(all_new_insights)} new insights, {sections_updated} sections updated")
+            else:
+                logger.info("No insights extracted from chat sessions")
+            
+            return canvas
+            
+        except Exception as e:
+            logger.error(f"Error updating canvas for user {user_id}: {e}")
+            import traceback
+            logger.error(f"Full traceback: {traceback.format_exc()}")
+            raise
+    
+    async def _get_all_user_chat_sessions(self, user_id: str) -> List[Dict]:
+        """Get all chat sessions for a user"""
+        try:
+            db = self.get_database()
+            user_object_id = ObjectId(user_id)
+            
+            cursor = db.chat_sessions.find({
+                "user_id": user_object_id,
+                "is_active": {"$ne": False},  # Include active sessions
+                "deleted_at": {"$exists": False}  # Exclude deleted sessions
+            }).sort("created_at", -1)  # Most recent first
+            
+            chat_sessions = await cursor.to_list(length=100)  # Limit to last 100 chats
+            return chat_sessions
+            
+        except Exception as e:
+            logger.error(f"Error getting all chat sessions for user {user_id}: {e}")
+            return []
+    
+    async def _get_new_chat_sessions(self, user_id: str, since: Optional[datetime]) -> List[Dict]:
+        """Get chat sessions created or updated after a specific time"""
+        try:
+            db = self.get_database()
+            user_object_id = ObjectId(user_id)
+            
+            # Build query filter
+            query_filter = {
+                "user_id": user_object_id,
+                "is_active": {"$ne": False},
+                "deleted_at": {"$exists": False}
+            }
+            
+            if since:
+                # Get chats created or updated after the since timestamp
+                query_filter["$or"] = [
+                    {"created_at": {"$gt": since}},
+                    {"updated_at": {"$gt": since}}
+                ]
+            else:
+                # If no since time, get chats from last 7 days as default
+                one_week_ago = datetime.utcnow() - timedelta(days=7)
+                query_filter["created_at"] = {"$gte": one_week_ago}
+            
+            cursor = db.chat_sessions.find(query_filter).sort("created_at", -1)
+            chat_sessions = await cursor.to_list(length=50)  # Limit for incremental updates
+            
+            return chat_sessions
+            
+        except Exception as e:
+            logger.error(f"Error getting new chat sessions for user {user_id}: {e}")
+            return []
+    
+    async def _save_canvas(self, canvas: PhdCanvas):
+        """Save canvas to database"""
+        try:
+            db = self.get_database()
+            
+            # Update existing canvas
+            await db.phd_canvases.replace_one(
+                {"_id": canvas.id},
+                canvas.dict(by_alias=True),
+                upsert=True
+            )
+            
+            logger.info(f"Saved canvas {canvas.id} to database")
+            
+        except Exception as e:
+            logger.error(f"Error saving canvas to database: {e}")
+            raise
+    
+    async def delete_canvas(self, user_id: str) -> bool:
+        """Delete canvas for a user"""
+        try:
+            db = self.get_database()
+            user_object_id = ObjectId(user_id)
+            
+            result = await db.phd_canvases.delete_one({"user_id": user_object_id})
+            
+            if result.deleted_count > 0:
+                logger.info(f"Deleted canvas for user {user_id}")
+                return True
+            else:
+                logger.info(f"No canvas found to delete for user {user_id}")
+                return False
+                
+        except Exception as e:
+            logger.error(f"Error deleting canvas for user {user_id}: {e}")
+            return False
+    
+    async def get_canvas_stats(self, user_id: str) -> Dict:
+        """Get statistics about user's canvas"""
+        try:
+            canvas = await self.get_or_create_canvas(user_id)
+            
+            stats = {
+                "total_insights": canvas.total_insights,
+                "total_sections": len(canvas.sections),
+                "last_updated": canvas.last_updated,
+                "last_chat_processed": canvas.last_chat_processed,
+                "created_at": canvas.created_at,
+                "auto_update": canvas.auto_update,
+                "sections_breakdown": {}
+            }
+            
+            # Add breakdown by section
+            for section_key, section in canvas.sections.items():
+                stats["sections_breakdown"][section_key] = {
+                    "title": section.title,
+                    "insight_count": len(section.insights),
+                    "last_updated": section.updated_at,
+                    "priority": section.priority
+                }
+            
+            return stats
+            
+        except Exception as e:
+            logger.error(f"Error getting canvas stats for user {user_id}: {e}")
+            return {
+                "total_insights": 0,
+                "total_sections": 0,
+                "error": str(e)
+            }
+    
+    async def export_canvas_for_printing(self, user_id: str) -> Dict:
+        """Export canvas in a print-optimized format"""
+        try:
+            canvas = await self.get_or_create_canvas(user_id)
+            
+            # Sort sections by priority and insight count
+            sorted_sections = []
+            for section_key, section in canvas.sections.items():
+                if section.insights:  # Only include sections with insights
+                    # Sort insights by confidence score
+                    sorted_insights = sorted(
+                        section.insights, 
+                        key=lambda x: x.confidence_score, 
+                        reverse=True
+                    )
+                    
+                    sorted_sections.append({
+                        "key": section_key,
+                        "title": section.title,
+                        "description": section.description,
+                        "insights": [
+                            {
+                                "content": insight.content,
+                                "source": insight.source_persona.title(),
+                                "confidence": round(insight.confidence_score, 2)
+                            }
+                            for insight in sorted_insights[:8]  # Limit for printing
+                        ],
+                        "insight_count": len(section.insights),
+                        "priority": section.priority
+                    })
+            
+            # Sort sections by priority, then by insight count
+            sorted_sections.sort(key=lambda x: (x["priority"], -x["insight_count"]))
+            
+            return {
+                "user_id": user_id,
+                "generated_at": datetime.utcnow(),
+                "total_insights": canvas.total_insights,
+                "last_updated": canvas.last_updated,
+                "sections": sorted_sections,
+                "metadata": {
+                    "canvas_id": str(canvas.id),
+                    "created_at": canvas.created_at,
+                    "version": "1.0"
+                }
+            }
+            
+        except Exception as e:
+            logger.error(f"Error exporting canvas for printing: {e}")
+            raise
+
+# Global canvas manager instance
+canvas_manager = CanvasManager()
+
+def get_canvas_manager() -> CanvasManager:
+    """Get the global canvas manager instance"""
+    return canvas_manager

multi_llm_chatbot_backend/app/models/phd_canvas.py

@@ -0,0 +1,115 @@
+from pydantic import BaseModel, Field
+from typing import Dict, List, Optional, Any
+from datetime import datetime
+from bson import ObjectId
+from app.models.user import PyObjectId
+
+class CanvasInsight(BaseModel):
+    """Individual insight extracted from chat messages"""
+    content: str
+    source_persona: str
+    source_message_id: Optional[str] = None
+    source_chat_session: Optional[str] = None
+    confidence_score: float = Field(ge=0.0, le=1.0, default=0.8)
+    extracted_at: datetime = Field(default_factory=datetime.utcnow)
+    keywords: List[str] = Field(default_factory=list)
+
+class CanvasSection(BaseModel):
+    """A themed section of the PhD Canvas with related insights"""
+    title: str
+    description: str
+    insights: List[CanvasInsight] = Field(default_factory=list)
+    priority: int = Field(default=1, ge=1, le=5)  # 1=highest priority
+    updated_at: datetime = Field(default_factory=datetime.utcnow)
+
+class PhdCanvas(BaseModel):
+    """Main PhD Canvas model storing all user insights organized by sections"""
+    id: PyObjectId = Field(default_factory=PyObjectId, alias="_id")
+    user_id: PyObjectId
+    
+    # Canvas sections organized by theme
+    sections: Dict[str, CanvasSection] = Field(default_factory=dict)
+    
+    # Metadata
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    last_updated: datetime = Field(default_factory=datetime.utcnow)
+    last_chat_processed: Optional[datetime] = None
+    total_insights: int = Field(default=0)
+    
+    # Settings
+    auto_update: bool = Field(default=True)
+    print_optimized: bool = Field(default=True)
+    
+    class Config:
+        allow_population_by_field_name = True
+        arbitrary_types_allowed = True
+        json_encoders = {ObjectId: str}
+
+    def update_section(self, section_key: str, insights: List[CanvasInsight]):
+        """Update a specific canvas section with new insights"""
+        if section_key not in self.sections:
+            self.sections[section_key] = CanvasSection(
+                title=self._get_section_title(section_key),
+                description=self._get_section_description(section_key)
+            )
+        
+        # Add new insights and remove duplicates
+        existing_contents = {insight.content for insight in self.sections[section_key].insights}
+        new_insights = [insight for insight in insights if insight.content not in existing_contents]
+        
+        self.sections[section_key].insights.extend(new_insights)
+        self.sections[section_key].updated_at = datetime.utcnow()
+        self.last_updated = datetime.utcnow()
+        
+        # Update total insights count
+        self.total_insights = sum(len(section.insights) for section in self.sections.values())
+    
+    def _get_section_title(self, section_key: str) -> str:
+        """Get human-readable title for section"""
+        titles = {
+            "research_progress": "Research Progress & Milestones",
+            "methodology": "Research Methods & Approach", 
+            "theoretical_framework": "Theoretical Foundations",
+            "challenges_obstacles": "Challenges & Solutions",
+            "next_steps": "Action Items & Next Steps",
+            "writing_communication": "Writing & Communication",
+            "career_development": "Academic Career Planning",
+            "literature_review": "Literature & Sources",
+            "data_analysis": "Data & Analysis",
+            "motivation_mindset": "Motivation & Mindset"
+        }
+        return titles.get(section_key, section_key.replace("_", " ").title())
+    
+    def _get_section_description(self, section_key: str) -> str:
+        """Get description for each section"""
+        descriptions = {
+            "research_progress": "Key milestones, accomplishments, and timeline updates",
+            "methodology": "Research design decisions and methodological insights",
+            "theoretical_framework": "Theoretical perspectives and conceptual foundations",
+            "challenges_obstacles": "Challenges faced and strategies for overcoming them",
+            "next_steps": "Immediate action items and upcoming priorities",
+            "writing_communication": "Writing strategies and communication insights",
+            "career_development": "Professional development and career planning",
+            "literature_review": "Literature gaps, sources, and review strategies",
+            "data_analysis": "Data collection and analysis approaches",
+            "motivation_mindset": "Motivational insights and mental health considerations"
+        }
+        return descriptions.get(section_key, "General insights and guidance")
+
+class CanvasResponse(BaseModel):
+    """Response model for canvas API endpoints"""
+    id: str
+    user_id: str
+    sections: Dict[str, CanvasSection]
+    created_at: datetime
+    last_updated: datetime
+    last_chat_processed: Optional[datetime]
+    total_insights: int
+    auto_update: bool
+    print_optimized: bool
+
+class UpdateCanvasRequest(BaseModel):
+    """Request model for updating canvas"""
+    force_full_update: bool = Field(default=False)
+    include_chat_sessions: Optional[List[str]] = None  # Specific sessions to include
+    exclude_sections: Optional[List[str]] = None  # Sections to skip updating