aiverse / app /services /rag /source_manager.py
junaidte14's picture
updated the RAG System
ea1beca
Raw History Blame Contribute Delete
10.9 kB
"""
Enhanced RAG Source Manager with Real-Time Progress
Production-ready with proper DB session handling
"""
from typing import Optional, Dict, Any, List
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy import delete as sql_delete
from datetime import datetime, timedelta
import uuid
from app.db.models.rag_source import (
RAGSource,
SourceType,
SyncStatus,
)
from app.schemas.rag_source import GitHubSourceCreate
from app.db.models.user import User
from app.services.rag.vector_store import VectorStoreService
from app.services.rag.github_integration import GitHubRAGIntegration
class RAGSourceManager:
"""
Manage RAG sources with real-time sync progress.
"""
def __init__(self, db: AsyncSession, user: User):
self.db = db
self.user = user
self.vector_store = VectorStoreService()
self.github_integration = GitHubRAGIntegration()
async def initialize_infrastructure(self):
"""Ensure pgvector extension and HNSW indexes are ready."""
await self.vector_store.initialize(self.db)
async def list_sources(
self, collection_name: Optional[str] = None
) -> List[RAGSource]:
"""List all RAG sources for the current user."""
query = select(RAGSource).where(RAGSource.user_id == self.user.id)
if collection_name:
query = query.where(RAGSource.collection_name == collection_name)
result = await self.db.execute(query)
return result.scalars().all()
async def add_github_source(self, data: GitHubSourceCreate):
"""Add GitHub source and trigger initial sync."""
# Validate repo first
repo_info = self.github_integration.get_repo_info(data.repo_url)
new_source = RAGSource(
user_id=self.user.id,
source_type=SourceType.GITHUB,
source_identifier=data.repo_url,
display_name=data.display_name or repo_info.get("full_name"),
collection_name=data.collection_name,
auto_sync=data.auto_sync,
sync_interval_hours=data.sync_interval_hours,
config={
"branch": data.branch or repo_info.get("default_branch"),
"file_extensions": data.file_extensions or [],
},
last_commit_sha=repo_info.get("last_commit_sha"),
)
self.db.add(new_source)
await self.db.commit()
await self.db.refresh(new_source)
return new_source
async def sync_source_with_progress(
self, source_id: int, operation_id: str, db_session: AsyncSession
) -> Dict[str, Any]:
"""
Synchronize source with real-time progress updates.
Uses a fresh DB session for background operations.
"""
from app.services.rag.rag_service import progress_tracker
# Initialize infrastructure
await progress_tracker.update_progress(
operation_id, 0, "Initializing vector database...", {"source_id": source_id}
)
await self.initialize_infrastructure()
# Fetch source with the provided session
result = await db_session.execute(
select(RAGSource).where(RAGSource.id == source_id)
)
source = result.scalar_one_or_none()
if not source:
await progress_tracker.complete_operation(
operation_id, False, f"Source {source_id} not found"
)
raise ValueError(f"Source {source_id} not found")
# Update status to in-progress
source.last_sync_status = SyncStatus.IN_PROGRESS
await db_session.commit()
try:
if source.source_type == SourceType.GITHUB:
results = await self._sync_github_with_progress(
source, operation_id, db_session
)
else:
raise ValueError(f"Unsupported source type: {source.source_type}")
# Update source telemetry
source.last_sync_status = SyncStatus.COMPLETED
source.last_sync_at = datetime.utcnow()
source.last_sync_error = None
source.total_files = results["files_processed"]
source.total_chunks = results["chunks_created"]
if "commit_sha" in results:
source.last_commit_sha = results["commit_sha"]
await db_session.commit()
await progress_tracker.complete_operation(operation_id, True)
return results
except Exception as e:
source.last_sync_status = SyncStatus.FAILED
source.last_sync_error = self.safe_error_message(e)
await db_session.commit()
await progress_tracker.complete_operation(operation_id, False, str(e))
raise
@staticmethod
def safe_error_message(e: Exception, max_length: int = 1000) -> str:
"""Safely truncate error messages for database storage."""
msg = str(e)
return msg[:max_length]
async def _sync_github_with_progress(
self, source: RAGSource, operation_id: str, db_session: AsyncSession
) -> Dict[str, Any]:
"""
Perform GitHub sync with real-time progress updates.
"""
from app.services.rag.rag_service import progress_tracker
config = source.config or {}
try:
# Step 1: Delete old documents
await progress_tracker.update_progress(
operation_id,
1,
"Cleaning up old documents...",
{"source_id": source.id},
)
await self._delete_source_documents(source, db_session)
await progress_tracker.update_progress(
operation_id,
2,
"Old documents deleted successfully",
{"source_id": source.id},
)
# Step 2: Clone/update repository
await progress_tracker.update_progress(
operation_id,
3,
f"Cloning repository: {source.source_identifier}",
{"repo_url": source.source_identifier},
)
repo_path, documents = self.github_integration.process_repository(
repo_url=source.source_identifier,
branch=config.get("branch"),
file_extensions=config.get("file_extensions"),
metadata={
"source_id": source.id,
"repository": source.display_name,
"ecosystem": "versana",
"user_id": self.user.id,
},
)
await progress_tracker.update_progress(
operation_id,
4,
f"Repository cloned. Found {len(documents)} chunks to process",
{
"chunk_count": len(documents),
"unique_files": len(
set(doc["metadata"]["source"] for doc in documents)
),
},
)
# Step 3: Store documents with progress
if documents:
await progress_tracker.update_progress(
operation_id,
5,
f"Starting to store {len(documents)} chunks...",
{"chunk_count": len(documents)},
)
# This will send incremental progress updates
await self.vector_store.add_documents_with_progress(
collection_name=source.collection_name,
documents=[doc["text"] for doc in documents],
metadatas=[doc["metadata"] for doc in documents],
ids=[doc["id"] for doc in documents],
db_session=db_session,
operation_id=operation_id,
)
else:
await progress_tracker.update_progress(
operation_id, 5, "No documents to store", {"chunk_count": 0}
)
commit_sha = self.github_integration.get_last_commit_sha(repo_path)
return {
"files_processed": len(
set(doc["metadata"]["source"] for doc in documents)
),
"chunks_created": len(documents),
"commit_sha": commit_sha,
}
except Exception as e:
await progress_tracker.update_progress(
operation_id,
-1,
f"Sync failed: {str(e)}",
{"error": str(e), "source_id": source.id},
)
raise
async def _delete_source_documents(
self, source: RAGSource, db_session: AsyncSession
) -> None:
"""Delete all vector embeddings associated with a specific source."""
from sqlalchemy import delete
from app.services.rag.pgvector_store import DocumentEmbedding
stmt = delete(DocumentEmbedding).where(
DocumentEmbedding.collection_name == source.collection_name,
DocumentEmbedding.metadata_.contains({"source_id": source.id}),
)
await db_session.execute(stmt)
await db_session.commit()
async def delete_source(self, source_id: int) -> None:
"""Delete source record and all associated embeddings."""
result = await self.db.execute(
select(RAGSource).where(
RAGSource.id == source_id, RAGSource.user_id == self.user.id
)
)
source = result.scalar_one_or_none()
if not source:
raise ValueError(f"Source {source_id} not found")
# Delete all vector embeddings for this source
await self._delete_source_documents(source, self.db)
# Delete the source record
await self.db.delete(source)
await self.db.commit()
async def get_sources_needing_sync(self) -> List[RAGSource]:
"""Get all sources that are due for sync."""
query = select(RAGSource).where(
RAGSource.user_id == self.user.id,
RAGSource.auto_sync == True,
)
result = await self.db.execute(query)
all_sources = result.scalars().all()
sources_to_sync = []
for source in all_sources:
if self._should_sync(source):
sources_to_sync.append(source)
return sources_to_sync
def _should_sync(self, source: RAGSource) -> bool:
"""Determine if a source should be synced."""
if not source.last_sync_at:
return True
sync_interval = timedelta(hours=source.sync_interval_hours)
time_since_sync = datetime.utcnow() - source.last_sync_at
return time_since_sync >= sync_interval