Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /ViperJuice_Code-Index-MCP /scripts /index_remaining_repos.py
| #!/usr/bin/env python3 | |
| """ | |
| Index remaining repositories, skipping very large ones. | |
| Focuses on completing small to medium repositories first. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import logging | |
| from pathlib import Path | |
| from datetime import datetime | |
| from mcp_server.core.path_utils import PathUtils | |
| # Add project root to Python path | |
| sys.path.insert(0, str(Path(__file__).parent.parent)) | |
| # Configure logging | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format='%(asctime)s - %(levelname)s - %(message)s' | |
| ) | |
| logger = logging.getLogger(__name__) | |
| # Large repositories to skip | |
| SKIP_LARGE_REPOS = { | |
| 'grpc', 'TypeScript', 'react', 'django', 'aspnetcore', | |
| 'rails', 'laravel', 'spring-boot' | |
| } | |
| def get_progress(): | |
| """Load current progress.""" | |
| progress_file = "semantic_indexing_progress.json" | |
| if Path(progress_file).exists(): | |
| with open(progress_file, 'r') as f: | |
| progress = json.load(f) | |
| return progress.get("completed", {}), progress.get("in_progress") | |
| return {}, None | |
| def get_all_repos(): | |
| """Get list of all repositories.""" | |
| repos = [] | |
| test_repos_dir = PathUtils.get_workspace_root() / "test_repos" | |
| for repo_path in test_repos_dir.rglob('.git'): | |
| if repo_path.is_dir(): | |
| repo_dir = repo_path.parent | |
| repo_name = repo_dir.name | |
| repos.append(repo_name) | |
| return repos | |
| def main(): | |
| """Main function to show remaining work.""" | |
| completed, in_progress = get_progress() | |
| all_repos = get_all_repos() | |
| logger.info("Semantic Indexing Progress Report") | |
| logger.info("="*60) | |
| # Completed repositories | |
| logger.info(f"\n✅ Completed ({len(completed)} repositories):") | |
| total_files = 0 | |
| total_embeddings = 0 | |
| for key, info in completed.items(): | |
| total_files += info['files_processed'] | |
| total_embeddings += info['embeddings'] | |
| logger.info(f" {info['repo']}: {info['files_processed']} files → {info['embeddings']:,} embeddings") | |
| logger.info(f"\nTotal: {total_files:,} files → {total_embeddings:,} embeddings") | |
| # In progress | |
| if in_progress: | |
| logger.info(f"\n🔄 In Progress:") | |
| logger.info(f" {in_progress['repo_key']}: {len(in_progress['processed_files'])} files processed so far") | |
| # Remaining repositories | |
| completed_names = {info['repo'] for info in completed.values()} | |
| if in_progress: | |
| completed_names.add(in_progress['repo_key'].split('_', 1)[1]) | |
| remaining = [r for r in all_repos if r not in completed_names] | |
| logger.info(f"\n📋 Remaining ({len(remaining)} repositories):") | |
| # Categorize by size | |
| small_repos = [] | |
| large_repos = [] | |
| for repo in remaining: | |
| if repo in SKIP_LARGE_REPOS: | |
| large_repos.append(repo) | |
| else: | |
| small_repos.append(repo) | |
| if small_repos: | |
| logger.info(f"\n Small/Medium (ready to process): {len(small_repos)}") | |
| for repo in sorted(small_repos): | |
| logger.info(f" - {repo}") | |
| if large_repos: | |
| logger.info(f"\n Large (requires dedicated processing): {len(large_repos)}") | |
| for repo in sorted(large_repos): | |
| logger.info(f" - {repo} ⚠️") | |
| # Next steps | |
| logger.info("\n" + "="*60) | |
| logger.info("Next Steps:") | |
| logger.info("1. Continue running: python scripts/index_all_repos_complete.py") | |
| logger.info("2. It will resume from where it left off") | |
| logger.info(f"3. Consider skipping large repos: {', '.join(sorted(large_repos))}") | |
| # Save summary | |
| summary = { | |
| "timestamp": datetime.now().isoformat(), | |
| "completed": len(completed), | |
| "total_files": total_files, | |
| "total_embeddings": total_embeddings, | |
| "remaining": len(remaining), | |
| "remaining_small": len(small_repos), | |
| "remaining_large": len(large_repos), | |
| "repositories": { | |
| "completed": list(completed_names), | |
| "remaining_small": sorted(small_repos), | |
| "remaining_large": sorted(large_repos) | |
| } | |
| } | |
| with open("indexing_progress_summary.json", 'w') as f: | |
| json.dump(summary, f, indent=2) | |
| logger.info(f"\nSummary saved to: indexing_progress_summary.json") | |
| if __name__ == "__main__": | |
| main() |
Xet Storage Details
- Size:
- 4.32 kB
- Xet hash:
- 78f4334bafe0420c62ff3fece4befe0adc5b5051e40209bdc96f914d41459f8c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.