""" Enhanced RAG Source Manager with Real-Time Progress Production-ready with proper DB session handling """ from typing import Optional, Dict, Any, List from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy import delete as sql_delete from datetime import datetime, timedelta import uuid from app.db.models.rag_source import ( RAGSource, SourceType, SyncStatus, ) from app.schemas.rag_source import GitHubSourceCreate from app.db.models.user import User from app.services.rag.vector_store import VectorStoreService from app.services.rag.github_integration import GitHubRAGIntegration class RAGSourceManager: """ Manage RAG sources with real-time sync progress. """ def __init__(self, db: AsyncSession, user: User): self.db = db self.user = user self.vector_store = VectorStoreService() self.github_integration = GitHubRAGIntegration() async def initialize_infrastructure(self): """Ensure pgvector extension and HNSW indexes are ready.""" await self.vector_store.initialize(self.db) async def list_sources( self, collection_name: Optional[str] = None ) -> List[RAGSource]: """List all RAG sources for the current user.""" query = select(RAGSource).where(RAGSource.user_id == self.user.id) if collection_name: query = query.where(RAGSource.collection_name == collection_name) result = await self.db.execute(query) return result.scalars().all() async def add_github_source(self, data: GitHubSourceCreate): """Add GitHub source and trigger initial sync.""" # Validate repo first repo_info = self.github_integration.get_repo_info(data.repo_url) new_source = RAGSource( user_id=self.user.id, source_type=SourceType.GITHUB, source_identifier=data.repo_url, display_name=data.display_name or repo_info.get("full_name"), collection_name=data.collection_name, auto_sync=data.auto_sync, sync_interval_hours=data.sync_interval_hours, config={ "branch": data.branch or repo_info.get("default_branch"), "file_extensions": data.file_extensions or [], }, last_commit_sha=repo_info.get("last_commit_sha"), ) self.db.add(new_source) await self.db.commit() await self.db.refresh(new_source) return new_source async def sync_source_with_progress( self, source_id: int, operation_id: str, db_session: AsyncSession ) -> Dict[str, Any]: """ Synchronize source with real-time progress updates. Uses a fresh DB session for background operations. """ from app.services.rag.rag_service import progress_tracker # Initialize infrastructure await progress_tracker.update_progress( operation_id, 0, "Initializing vector database...", {"source_id": source_id} ) await self.initialize_infrastructure() # Fetch source with the provided session result = await db_session.execute( select(RAGSource).where(RAGSource.id == source_id) ) source = result.scalar_one_or_none() if not source: await progress_tracker.complete_operation( operation_id, False, f"Source {source_id} not found" ) raise ValueError(f"Source {source_id} not found") # Update status to in-progress source.last_sync_status = SyncStatus.IN_PROGRESS await db_session.commit() try: if source.source_type == SourceType.GITHUB: results = await self._sync_github_with_progress( source, operation_id, db_session ) else: raise ValueError(f"Unsupported source type: {source.source_type}") # Update source telemetry source.last_sync_status = SyncStatus.COMPLETED source.last_sync_at = datetime.utcnow() source.last_sync_error = None source.total_files = results["files_processed"] source.total_chunks = results["chunks_created"] if "commit_sha" in results: source.last_commit_sha = results["commit_sha"] await db_session.commit() await progress_tracker.complete_operation(operation_id, True) return results except Exception as e: source.last_sync_status = SyncStatus.FAILED source.last_sync_error = self.safe_error_message(e) await db_session.commit() await progress_tracker.complete_operation(operation_id, False, str(e)) raise @staticmethod def safe_error_message(e: Exception, max_length: int = 1000) -> str: """Safely truncate error messages for database storage.""" msg = str(e) return msg[:max_length] async def _sync_github_with_progress( self, source: RAGSource, operation_id: str, db_session: AsyncSession ) -> Dict[str, Any]: """ Perform GitHub sync with real-time progress updates. """ from app.services.rag.rag_service import progress_tracker config = source.config or {} try: # Step 1: Delete old documents await progress_tracker.update_progress( operation_id, 1, "Cleaning up old documents...", {"source_id": source.id}, ) await self._delete_source_documents(source, db_session) await progress_tracker.update_progress( operation_id, 2, "Old documents deleted successfully", {"source_id": source.id}, ) # Step 2: Clone/update repository await progress_tracker.update_progress( operation_id, 3, f"Cloning repository: {source.source_identifier}", {"repo_url": source.source_identifier}, ) repo_path, documents = self.github_integration.process_repository( repo_url=source.source_identifier, branch=config.get("branch"), file_extensions=config.get("file_extensions"), metadata={ "source_id": source.id, "repository": source.display_name, "ecosystem": "versana", "user_id": self.user.id, }, ) await progress_tracker.update_progress( operation_id, 4, f"Repository cloned. Found {len(documents)} chunks to process", { "chunk_count": len(documents), "unique_files": len( set(doc["metadata"]["source"] for doc in documents) ), }, ) # Step 3: Store documents with progress if documents: await progress_tracker.update_progress( operation_id, 5, f"Starting to store {len(documents)} chunks...", {"chunk_count": len(documents)}, ) # This will send incremental progress updates await self.vector_store.add_documents_with_progress( collection_name=source.collection_name, documents=[doc["text"] for doc in documents], metadatas=[doc["metadata"] for doc in documents], ids=[doc["id"] for doc in documents], db_session=db_session, operation_id=operation_id, ) else: await progress_tracker.update_progress( operation_id, 5, "No documents to store", {"chunk_count": 0} ) commit_sha = self.github_integration.get_last_commit_sha(repo_path) return { "files_processed": len( set(doc["metadata"]["source"] for doc in documents) ), "chunks_created": len(documents), "commit_sha": commit_sha, } except Exception as e: await progress_tracker.update_progress( operation_id, -1, f"Sync failed: {str(e)}", {"error": str(e), "source_id": source.id}, ) raise async def _delete_source_documents( self, source: RAGSource, db_session: AsyncSession ) -> None: """Delete all vector embeddings associated with a specific source.""" from sqlalchemy import delete from app.services.rag.pgvector_store import DocumentEmbedding stmt = delete(DocumentEmbedding).where( DocumentEmbedding.collection_name == source.collection_name, DocumentEmbedding.metadata_.contains({"source_id": source.id}), ) await db_session.execute(stmt) await db_session.commit() async def delete_source(self, source_id: int) -> None: """Delete source record and all associated embeddings.""" result = await self.db.execute( select(RAGSource).where( RAGSource.id == source_id, RAGSource.user_id == self.user.id ) ) source = result.scalar_one_or_none() if not source: raise ValueError(f"Source {source_id} not found") # Delete all vector embeddings for this source await self._delete_source_documents(source, self.db) # Delete the source record await self.db.delete(source) await self.db.commit() async def get_sources_needing_sync(self) -> List[RAGSource]: """Get all sources that are due for sync.""" query = select(RAGSource).where( RAGSource.user_id == self.user.id, RAGSource.auto_sync == True, ) result = await self.db.execute(query) all_sources = result.scalars().all() sources_to_sync = [] for source in all_sources: if self._should_sync(source): sources_to_sync.append(source) return sources_to_sync def _should_sync(self, source: RAGSource) -> bool: """Determine if a source should be synced.""" if not source.last_sync_at: return True sync_interval = timedelta(hours=source.sync_interval_hours) time_since_sync = datetime.utcnow() - source.last_sync_at return time_since_sync >= sync_interval