""" GitHub Repository Integration for RAG Refined for Versana Ecosystem and JSONB Metadata """ import os import shutil from typing import Optional, Dict, Any, List from pathlib import Path import git from github import Github, GithubException import validators from app.services.rag.document_processor import DocumentProcessor class GitHubRAGIntegration: """ Handle GitHub repository integration for RAG. Optimized to feed structured metadata into pgvector. """ def __init__( self, github_token: Optional[str] = None, clone_base_path: str = "./data/github_repos", ): self.github_token = github_token or os.getenv("GITHUB_TOKEN") self.clone_base_path = Path(clone_base_path) self.clone_base_path.mkdir(parents=True, exist_ok=True) if self.github_token: self.github_client = Github(self.github_token) else: self.github_client = Github() def validate_repo_url(self, repo_url: str) -> Dict[str, str]: """Validate and parse GitHub repository URL.""" if not validators.url(repo_url): raise ValueError("Invalid URL format") if "github.com" not in repo_url: raise ValueError("Not a GitHub URL") parts = repo_url.rstrip("/").rstrip(".git").split("/") if repo_url.startswith("git@"): owner_repo = parts[-1].split(":")[-1] owner, repo = owner_repo.split("/") else: owner, repo = parts[-2], parts[-1] return {"owner": owner, "repo": repo} def get_repo_info(self, repo_url: str) -> Dict[str, Any]: """Fetch repository metadata from GitHub API.""" parsed = self.validate_repo_url(repo_url) try: repo = self.github_client.get_repo(f"{parsed['owner']}/{parsed['repo']}") default_branch = repo.default_branch # Get the latest commit SHA more reliably try: branch_obj = repo.get_branch(default_branch) last_commit_sha = branch_obj.commit.sha except Exception: # Fallback: get the latest commit from the default branch commits = repo.get_commits(sha=default_branch) last_commit_sha = commits[0].sha return { "full_name": repo.full_name, "description": repo.description or "", "default_branch": default_branch, "size": repo.size, "language": repo.language or "Unknown", "is_private": repo.private, "last_commit_sha": last_commit_sha, "last_updated": repo.updated_at.isoformat(), } except GithubException as e: raise ValueError(f"GitHub API error: {e.data.get('message', str(e)) if hasattr(e, 'data') else str(e)}") except Exception as e: raise ValueError(f"GitHub metadata error: {str(e)}") def clone_or_update_repo( self, repo_url: str, branch: Optional[str] = None, shallow: bool = True ) -> Path: """Clone or pull latest changes for the repository[cite: 13].""" parsed = self.validate_repo_url(repo_url) repo_name = f"{parsed['owner']}_{parsed['repo']}" repo_path = self.clone_base_path / repo_name if self.github_token and not repo_url.startswith("git@"): clone_url = repo_url.replace( "https://github.com", f"https://{self.github_token}@github.com" ) else: clone_url = repo_url try: if repo_path.exists(): repo = git.Repo(repo_path) origin = repo.remotes.origin origin.fetch() target = branch or repo.active_branch.name repo.git.checkout(target) repo.git.reset("--hard", f"origin/{target}") else: clone_kwargs = {"depth": 1} if shallow else {} if branch: clone_kwargs["branch"] = branch git.Repo.clone_from(clone_url, repo_path, **clone_kwargs) return repo_path except git.GitCommandError as e: raise RuntimeError(f"Git operation failed: {str(e)}") def process_repository( self, repo_url: str, branch: Optional[str] = None, file_extensions: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, metadata: Optional[Dict[str, Any]] = None, ) -> tuple[Path, List[Dict[str, Any]]]: """ Clone and process repository for pgvector JSONB ingestion[cite: 5, 13]. """ repo_path = self.clone_or_update_repo(repo_url, branch) repo_info = self.get_repo_info(repo_url) # Structured metadata for the Versana ecosystem full_metadata = { "source": "github", "repo_full_name": repo_info["full_name"], "repo_id": f"{repo_info['full_name']}_{branch or repo_info['default_branch']}", "last_commit": repo_info["last_commit_sha"], **(metadata or {}), } # Initialize processor with standard Versana chunking size[cite: 6] processor = DocumentProcessor(chunk_size=500, chunk_overlap=50) # Focus on core WordPress and Versana file types[cite: 13] if file_extensions is None: file_extensions = [ ".php", ".js", ".ts", ".tsx", ".md", ".json", ".css", ".scss", ".sql", ] # Ensure vendor and build folders are never indexed[cite: 13] default_excludes = [ "vendor", "node_modules", ".git", "dist", "build", "*.min.js", ] exclude_patterns = (exclude_patterns or []) + default_excludes documents = processor.process_directory( directory=str(repo_path), file_extensions=file_extensions, exclude_patterns=exclude_patterns, metadata=full_metadata, ) return repo_path, documents def get_last_commit_sha(self, repo_path: Path) -> str: """Get SHA from local clone to avoid extra API calls.""" repo = git.Repo(repo_path) return repo.head.object.hexsha