Spaces:
Runtime error
Runtime error
Download app/services/rag/source_manager.py from junaidte14/aiverse: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/spaces/junaidte14/aiverse/resolve/main/app/services/rag/source_manager.py
- Command line
-
hf download hf://spaces/junaidte14/aiverse/app/services/rag/source_manager.py
-
curl -L -o source_manager.py https://huggingface.co/spaces/junaidte14/aiverse/resolve/main/app/services/rag/source_manager.py
10.9 kB
| """ | |
| Enhanced RAG Source Manager with Real-Time Progress | |
| Production-ready with proper DB session handling | |
| """ | |
| from typing import Optional, Dict, Any, List | |
| from sqlalchemy.ext.asyncio import AsyncSession | |
| from sqlalchemy.future import select | |
| from sqlalchemy import delete as sql_delete | |
| from datetime import datetime, timedelta | |
| import uuid | |
| from app.db.models.rag_source import ( | |
| RAGSource, | |
| SourceType, | |
| SyncStatus, | |
| ) | |
| from app.schemas.rag_source import GitHubSourceCreate | |
| from app.db.models.user import User | |
| from app.services.rag.vector_store import VectorStoreService | |
| from app.services.rag.github_integration import GitHubRAGIntegration | |
| class RAGSourceManager: | |
| """ | |
| Manage RAG sources with real-time sync progress. | |
| """ | |
| def __init__(self, db: AsyncSession, user: User): | |
| self.db = db | |
| self.user = user | |
| self.vector_store = VectorStoreService() | |
| self.github_integration = GitHubRAGIntegration() | |
| async def initialize_infrastructure(self): | |
| """Ensure pgvector extension and HNSW indexes are ready.""" | |
| await self.vector_store.initialize(self.db) | |
| async def list_sources( | |
| self, collection_name: Optional[str] = None | |
| ) -> List[RAGSource]: | |
| """List all RAG sources for the current user.""" | |
| query = select(RAGSource).where(RAGSource.user_id == self.user.id) | |
| if collection_name: | |
| query = query.where(RAGSource.collection_name == collection_name) | |
| result = await self.db.execute(query) | |
| return result.scalars().all() | |
| async def add_github_source(self, data: GitHubSourceCreate): | |
| """Add GitHub source and trigger initial sync.""" | |
| # Validate repo first | |
| repo_info = self.github_integration.get_repo_info(data.repo_url) | |
| new_source = RAGSource( | |
| user_id=self.user.id, | |
| source_type=SourceType.GITHUB, | |
| source_identifier=data.repo_url, | |
| display_name=data.display_name or repo_info.get("full_name"), | |
| collection_name=data.collection_name, | |
| auto_sync=data.auto_sync, | |
| sync_interval_hours=data.sync_interval_hours, | |
| config={ | |
| "branch": data.branch or repo_info.get("default_branch"), | |
| "file_extensions": data.file_extensions or [], | |
| }, | |
| last_commit_sha=repo_info.get("last_commit_sha"), | |
| ) | |
| self.db.add(new_source) | |
| await self.db.commit() | |
| await self.db.refresh(new_source) | |
| return new_source | |
| async def sync_source_with_progress( | |
| self, source_id: int, operation_id: str, db_session: AsyncSession | |
| ) -> Dict[str, Any]: | |
| """ | |
| Synchronize source with real-time progress updates. | |
| Uses a fresh DB session for background operations. | |
| """ | |
| from app.services.rag.rag_service import progress_tracker | |
| # Initialize infrastructure | |
| await progress_tracker.update_progress( | |
| operation_id, 0, "Initializing vector database...", {"source_id": source_id} | |
| ) | |
| await self.initialize_infrastructure() | |
| # Fetch source with the provided session | |
| result = await db_session.execute( | |
| select(RAGSource).where(RAGSource.id == source_id) | |
| ) | |
| source = result.scalar_one_or_none() | |
| if not source: | |
| await progress_tracker.complete_operation( | |
| operation_id, False, f"Source {source_id} not found" | |
| ) | |
| raise ValueError(f"Source {source_id} not found") | |
| # Update status to in-progress | |
| source.last_sync_status = SyncStatus.IN_PROGRESS | |
| await db_session.commit() | |
| try: | |
| if source.source_type == SourceType.GITHUB: | |
| results = await self._sync_github_with_progress( | |
| source, operation_id, db_session | |
| ) | |
| else: | |
| raise ValueError(f"Unsupported source type: {source.source_type}") | |
| # Update source telemetry | |
| source.last_sync_status = SyncStatus.COMPLETED | |
| source.last_sync_at = datetime.utcnow() | |
| source.last_sync_error = None | |
| source.total_files = results["files_processed"] | |
| source.total_chunks = results["chunks_created"] | |
| if "commit_sha" in results: | |
| source.last_commit_sha = results["commit_sha"] | |
| await db_session.commit() | |
| await progress_tracker.complete_operation(operation_id, True) | |
| return results | |
| except Exception as e: | |
| source.last_sync_status = SyncStatus.FAILED | |
| source.last_sync_error = self.safe_error_message(e) | |
| await db_session.commit() | |
| await progress_tracker.complete_operation(operation_id, False, str(e)) | |
| raise | |
| def safe_error_message(e: Exception, max_length: int = 1000) -> str: | |
| """Safely truncate error messages for database storage.""" | |
| msg = str(e) | |
| return msg[:max_length] | |
| async def _sync_github_with_progress( | |
| self, source: RAGSource, operation_id: str, db_session: AsyncSession | |
| ) -> Dict[str, Any]: | |
| """ | |
| Perform GitHub sync with real-time progress updates. | |
| """ | |
| from app.services.rag.rag_service import progress_tracker | |
| config = source.config or {} | |
| try: | |
| # Step 1: Delete old documents | |
| await progress_tracker.update_progress( | |
| operation_id, | |
| 1, | |
| "Cleaning up old documents...", | |
| {"source_id": source.id}, | |
| ) | |
| await self._delete_source_documents(source, db_session) | |
| await progress_tracker.update_progress( | |
| operation_id, | |
| 2, | |
| "Old documents deleted successfully", | |
| {"source_id": source.id}, | |
| ) | |
| # Step 2: Clone/update repository | |
| await progress_tracker.update_progress( | |
| operation_id, | |
| 3, | |
| f"Cloning repository: {source.source_identifier}", | |
| {"repo_url": source.source_identifier}, | |
| ) | |
| repo_path, documents = self.github_integration.process_repository( | |
| repo_url=source.source_identifier, | |
| branch=config.get("branch"), | |
| file_extensions=config.get("file_extensions"), | |
| metadata={ | |
| "source_id": source.id, | |
| "repository": source.display_name, | |
| "ecosystem": "versana", | |
| "user_id": self.user.id, | |
| }, | |
| ) | |
| await progress_tracker.update_progress( | |
| operation_id, | |
| 4, | |
| f"Repository cloned. Found {len(documents)} chunks to process", | |
| { | |
| "chunk_count": len(documents), | |
| "unique_files": len( | |
| set(doc["metadata"]["source"] for doc in documents) | |
| ), | |
| }, | |
| ) | |
| # Step 3: Store documents with progress | |
| if documents: | |
| await progress_tracker.update_progress( | |
| operation_id, | |
| 5, | |
| f"Starting to store {len(documents)} chunks...", | |
| {"chunk_count": len(documents)}, | |
| ) | |
| # This will send incremental progress updates | |
| await self.vector_store.add_documents_with_progress( | |
| collection_name=source.collection_name, | |
| documents=[doc["text"] for doc in documents], | |
| metadatas=[doc["metadata"] for doc in documents], | |
| ids=[doc["id"] for doc in documents], | |
| db_session=db_session, | |
| operation_id=operation_id, | |
| ) | |
| else: | |
| await progress_tracker.update_progress( | |
| operation_id, 5, "No documents to store", {"chunk_count": 0} | |
| ) | |
| commit_sha = self.github_integration.get_last_commit_sha(repo_path) | |
| return { | |
| "files_processed": len( | |
| set(doc["metadata"]["source"] for doc in documents) | |
| ), | |
| "chunks_created": len(documents), | |
| "commit_sha": commit_sha, | |
| } | |
| except Exception as e: | |
| await progress_tracker.update_progress( | |
| operation_id, | |
| -1, | |
| f"Sync failed: {str(e)}", | |
| {"error": str(e), "source_id": source.id}, | |
| ) | |
| raise | |
| async def _delete_source_documents( | |
| self, source: RAGSource, db_session: AsyncSession | |
| ) -> None: | |
| """Delete all vector embeddings associated with a specific source.""" | |
| from sqlalchemy import delete | |
| from app.services.rag.pgvector_store import DocumentEmbedding | |
| stmt = delete(DocumentEmbedding).where( | |
| DocumentEmbedding.collection_name == source.collection_name, | |
| DocumentEmbedding.metadata_.contains({"source_id": source.id}), | |
| ) | |
| await db_session.execute(stmt) | |
| await db_session.commit() | |
| async def delete_source(self, source_id: int) -> None: | |
| """Delete source record and all associated embeddings.""" | |
| result = await self.db.execute( | |
| select(RAGSource).where( | |
| RAGSource.id == source_id, RAGSource.user_id == self.user.id | |
| ) | |
| ) | |
| source = result.scalar_one_or_none() | |
| if not source: | |
| raise ValueError(f"Source {source_id} not found") | |
| # Delete all vector embeddings for this source | |
| await self._delete_source_documents(source, self.db) | |
| # Delete the source record | |
| await self.db.delete(source) | |
| await self.db.commit() | |
| async def get_sources_needing_sync(self) -> List[RAGSource]: | |
| """Get all sources that are due for sync.""" | |
| query = select(RAGSource).where( | |
| RAGSource.user_id == self.user.id, | |
| RAGSource.auto_sync == True, | |
| ) | |
| result = await self.db.execute(query) | |
| all_sources = result.scalars().all() | |
| sources_to_sync = [] | |
| for source in all_sources: | |
| if self._should_sync(source): | |
| sources_to_sync.append(source) | |
| return sources_to_sync | |
| def _should_sync(self, source: RAGSource) -> bool: | |
| """Determine if a source should be synced.""" | |
| if not source.last_sync_at: | |
| return True | |
| sync_interval = timedelta(hours=source.sync_interval_hours) | |
| time_since_sync = datetime.utcnow() - source.last_sync_at | |
| return time_since_sync >= sync_interval | |