Spaces:
Runtime error
Runtime error
Download app/services/rag/github_integration.py from junaidte14/aiverse: direct link, hf CLI and curl.
- Browser
- Download file 6.47 kB
-
https://huggingface.co/spaces/junaidte14/aiverse/resolve/main/app/services/rag/github_integration.py
- Command line
-
hf download hf://spaces/junaidte14/aiverse/app/services/rag/github_integration.py
-
curl -L -o github_integration.py https://huggingface.co/spaces/junaidte14/aiverse/resolve/main/app/services/rag/github_integration.py
6.47 kB
| """ | |
| GitHub Repository Integration for RAG | |
| Refined for Versana Ecosystem and JSONB Metadata | |
| """ | |
| import os | |
| import shutil | |
| from typing import Optional, Dict, Any, List | |
| from pathlib import Path | |
| import git | |
| from github import Github, GithubException | |
| import validators | |
| from app.services.rag.document_processor import DocumentProcessor | |
| class GitHubRAGIntegration: | |
| """ | |
| Handle GitHub repository integration for RAG. | |
| Optimized to feed structured metadata into pgvector. | |
| """ | |
| def __init__( | |
| self, | |
| github_token: Optional[str] = None, | |
| clone_base_path: str = "./data/github_repos", | |
| ): | |
| self.github_token = github_token or os.getenv("GITHUB_TOKEN") | |
| self.clone_base_path = Path(clone_base_path) | |
| self.clone_base_path.mkdir(parents=True, exist_ok=True) | |
| if self.github_token: | |
| self.github_client = Github(self.github_token) | |
| else: | |
| self.github_client = Github() | |
| def validate_repo_url(self, repo_url: str) -> Dict[str, str]: | |
| """Validate and parse GitHub repository URL.""" | |
| if not validators.url(repo_url): | |
| raise ValueError("Invalid URL format") | |
| if "github.com" not in repo_url: | |
| raise ValueError("Not a GitHub URL") | |
| parts = repo_url.rstrip("/").rstrip(".git").split("/") | |
| if repo_url.startswith("git@"): | |
| owner_repo = parts[-1].split(":")[-1] | |
| owner, repo = owner_repo.split("/") | |
| else: | |
| owner, repo = parts[-2], parts[-1] | |
| return {"owner": owner, "repo": repo} | |
| def get_repo_info(self, repo_url: str) -> Dict[str, Any]: | |
| """Fetch repository metadata from GitHub API.""" | |
| parsed = self.validate_repo_url(repo_url) | |
| try: | |
| repo = self.github_client.get_repo(f"{parsed['owner']}/{parsed['repo']}") | |
| default_branch = repo.default_branch | |
| # Get the latest commit SHA more reliably | |
| try: | |
| branch_obj = repo.get_branch(default_branch) | |
| last_commit_sha = branch_obj.commit.sha | |
| except Exception: | |
| # Fallback: get the latest commit from the default branch | |
| commits = repo.get_commits(sha=default_branch) | |
| last_commit_sha = commits[0].sha | |
| return { | |
| "full_name": repo.full_name, | |
| "description": repo.description or "", | |
| "default_branch": default_branch, | |
| "size": repo.size, | |
| "language": repo.language or "Unknown", | |
| "is_private": repo.private, | |
| "last_commit_sha": last_commit_sha, | |
| "last_updated": repo.updated_at.isoformat(), | |
| } | |
| except GithubException as e: | |
| raise ValueError(f"GitHub API error: {e.data.get('message', str(e)) if hasattr(e, 'data') else str(e)}") | |
| except Exception as e: | |
| raise ValueError(f"GitHub metadata error: {str(e)}") | |
| def clone_or_update_repo( | |
| self, repo_url: str, branch: Optional[str] = None, shallow: bool = True | |
| ) -> Path: | |
| """Clone or pull latest changes for the repository[cite: 13].""" | |
| parsed = self.validate_repo_url(repo_url) | |
| repo_name = f"{parsed['owner']}_{parsed['repo']}" | |
| repo_path = self.clone_base_path / repo_name | |
| if self.github_token and not repo_url.startswith("git@"): | |
| clone_url = repo_url.replace( | |
| "https://github.com", f"https://{self.github_token}@github.com" | |
| ) | |
| else: | |
| clone_url = repo_url | |
| try: | |
| if repo_path.exists(): | |
| repo = git.Repo(repo_path) | |
| origin = repo.remotes.origin | |
| origin.fetch() | |
| target = branch or repo.active_branch.name | |
| repo.git.checkout(target) | |
| repo.git.reset("--hard", f"origin/{target}") | |
| else: | |
| clone_kwargs = {"depth": 1} if shallow else {} | |
| if branch: | |
| clone_kwargs["branch"] = branch | |
| git.Repo.clone_from(clone_url, repo_path, **clone_kwargs) | |
| return repo_path | |
| except git.GitCommandError as e: | |
| raise RuntimeError(f"Git operation failed: {str(e)}") | |
| def process_repository( | |
| self, | |
| repo_url: str, | |
| branch: Optional[str] = None, | |
| file_extensions: Optional[List[str]] = None, | |
| exclude_patterns: Optional[List[str]] = None, | |
| metadata: Optional[Dict[str, Any]] = None, | |
| ) -> tuple[Path, List[Dict[str, Any]]]: | |
| """ | |
| Clone and process repository for pgvector JSONB ingestion[cite: 5, 13]. | |
| """ | |
| repo_path = self.clone_or_update_repo(repo_url, branch) | |
| repo_info = self.get_repo_info(repo_url) | |
| # Structured metadata for the Versana ecosystem | |
| full_metadata = { | |
| "source": "github", | |
| "repo_full_name": repo_info["full_name"], | |
| "repo_id": f"{repo_info['full_name']}_{branch or repo_info['default_branch']}", | |
| "last_commit": repo_info["last_commit_sha"], | |
| **(metadata or {}), | |
| } | |
| # Initialize processor with standard Versana chunking size[cite: 6] | |
| processor = DocumentProcessor(chunk_size=500, chunk_overlap=50) | |
| # Focus on core WordPress and Versana file types[cite: 13] | |
| if file_extensions is None: | |
| file_extensions = [ | |
| ".php", | |
| ".js", | |
| ".ts", | |
| ".tsx", | |
| ".md", | |
| ".json", | |
| ".css", | |
| ".scss", | |
| ".sql", | |
| ] | |
| # Ensure vendor and build folders are never indexed[cite: 13] | |
| default_excludes = [ | |
| "vendor", | |
| "node_modules", | |
| ".git", | |
| "dist", | |
| "build", | |
| "*.min.js", | |
| ] | |
| exclude_patterns = (exclude_patterns or []) + default_excludes | |
| documents = processor.process_directory( | |
| directory=str(repo_path), | |
| file_extensions=file_extensions, | |
| exclude_patterns=exclude_patterns, | |
| metadata=full_metadata, | |
| ) | |
| return repo_path, documents | |
| def get_last_commit_sha(self, repo_path: Path) -> str: | |
| """Get SHA from local clone to avoid extra API calls.""" | |
| repo = git.Repo(repo_path) | |
| return repo.head.object.hexsha |