aiverse / app /services /rag /github_integration.py
junaidte14's picture
aiverse backend to hf
3821102
Raw History Blame Contribute Delete
6.47 kB
"""
GitHub Repository Integration for RAG
Refined for Versana Ecosystem and JSONB Metadata
"""
import os
import shutil
from typing import Optional, Dict, Any, List
from pathlib import Path
import git
from github import Github, GithubException
import validators
from app.services.rag.document_processor import DocumentProcessor
class GitHubRAGIntegration:
"""
Handle GitHub repository integration for RAG.
Optimized to feed structured metadata into pgvector.
"""
def __init__(
self,
github_token: Optional[str] = None,
clone_base_path: str = "./data/github_repos",
):
self.github_token = github_token or os.getenv("GITHUB_TOKEN")
self.clone_base_path = Path(clone_base_path)
self.clone_base_path.mkdir(parents=True, exist_ok=True)
if self.github_token:
self.github_client = Github(self.github_token)
else:
self.github_client = Github()
def validate_repo_url(self, repo_url: str) -> Dict[str, str]:
"""Validate and parse GitHub repository URL."""
if not validators.url(repo_url):
raise ValueError("Invalid URL format")
if "github.com" not in repo_url:
raise ValueError("Not a GitHub URL")
parts = repo_url.rstrip("/").rstrip(".git").split("/")
if repo_url.startswith("git@"):
owner_repo = parts[-1].split(":")[-1]
owner, repo = owner_repo.split("/")
else:
owner, repo = parts[-2], parts[-1]
return {"owner": owner, "repo": repo}
def get_repo_info(self, repo_url: str) -> Dict[str, Any]:
"""Fetch repository metadata from GitHub API."""
parsed = self.validate_repo_url(repo_url)
try:
repo = self.github_client.get_repo(f"{parsed['owner']}/{parsed['repo']}")
default_branch = repo.default_branch
# Get the latest commit SHA more reliably
try:
branch_obj = repo.get_branch(default_branch)
last_commit_sha = branch_obj.commit.sha
except Exception:
# Fallback: get the latest commit from the default branch
commits = repo.get_commits(sha=default_branch)
last_commit_sha = commits[0].sha
return {
"full_name": repo.full_name,
"description": repo.description or "",
"default_branch": default_branch,
"size": repo.size,
"language": repo.language or "Unknown",
"is_private": repo.private,
"last_commit_sha": last_commit_sha,
"last_updated": repo.updated_at.isoformat(),
}
except GithubException as e:
raise ValueError(f"GitHub API error: {e.data.get('message', str(e)) if hasattr(e, 'data') else str(e)}")
except Exception as e:
raise ValueError(f"GitHub metadata error: {str(e)}")
def clone_or_update_repo(
self, repo_url: str, branch: Optional[str] = None, shallow: bool = True
) -> Path:
"""Clone or pull latest changes for the repository[cite: 13]."""
parsed = self.validate_repo_url(repo_url)
repo_name = f"{parsed['owner']}_{parsed['repo']}"
repo_path = self.clone_base_path / repo_name
if self.github_token and not repo_url.startswith("git@"):
clone_url = repo_url.replace(
"https://github.com", f"https://{self.github_token}@github.com"
)
else:
clone_url = repo_url
try:
if repo_path.exists():
repo = git.Repo(repo_path)
origin = repo.remotes.origin
origin.fetch()
target = branch or repo.active_branch.name
repo.git.checkout(target)
repo.git.reset("--hard", f"origin/{target}")
else:
clone_kwargs = {"depth": 1} if shallow else {}
if branch:
clone_kwargs["branch"] = branch
git.Repo.clone_from(clone_url, repo_path, **clone_kwargs)
return repo_path
except git.GitCommandError as e:
raise RuntimeError(f"Git operation failed: {str(e)}")
def process_repository(
self,
repo_url: str,
branch: Optional[str] = None,
file_extensions: Optional[List[str]] = None,
exclude_patterns: Optional[List[str]] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> tuple[Path, List[Dict[str, Any]]]:
"""
Clone and process repository for pgvector JSONB ingestion[cite: 5, 13].
"""
repo_path = self.clone_or_update_repo(repo_url, branch)
repo_info = self.get_repo_info(repo_url)
# Structured metadata for the Versana ecosystem
full_metadata = {
"source": "github",
"repo_full_name": repo_info["full_name"],
"repo_id": f"{repo_info['full_name']}_{branch or repo_info['default_branch']}",
"last_commit": repo_info["last_commit_sha"],
**(metadata or {}),
}
# Initialize processor with standard Versana chunking size[cite: 6]
processor = DocumentProcessor(chunk_size=500, chunk_overlap=50)
# Focus on core WordPress and Versana file types[cite: 13]
if file_extensions is None:
file_extensions = [
".php",
".js",
".ts",
".tsx",
".md",
".json",
".css",
".scss",
".sql",
]
# Ensure vendor and build folders are never indexed[cite: 13]
default_excludes = [
"vendor",
"node_modules",
".git",
"dist",
"build",
"*.min.js",
]
exclude_patterns = (exclude_patterns or []) + default_excludes
documents = processor.process_directory(
directory=str(repo_path),
file_extensions=file_extensions,
exclude_patterns=exclude_patterns,
metadata=full_metadata,
)
return repo_path, documents
def get_last_commit_sha(self, repo_path: Path) -> str:
"""Get SHA from local clone to avoid extra API calls."""
repo = git.Repo(repo_path)
return repo.head.object.hexsha