Buckets:

download
raw
4.91 kB
#!/usr/bin/env python3
"""Example usage of repository-level processing.
This example shows how to use the repository processing features
for large-scale code analysis.
"""
from pathlib import Path
from chunker.repo import GitAwareProcessorImpl, RepoProcessorImpl
from chunker.repo.patterns import load_gitignore_patterns
def process_repository_example():
"""Example: Process an entire repository with Git awareness."""
# Initialize processor with optimized settings for large repos
processor = RepoProcessorImpl(
max_workers=8, # Use 8 parallel workers
use_multiprocessing=False, # Use threads (faster for I/O)
chunk_batch_size=100, # Process 100 files per batch
memory_limit_mb=2048, # Limit memory to 2GB
)
# Process a repository (incremental mode)
repo_path = "/path/to/your/repository"
# This would process only changed files since last run
result = processor.process_repository(
repo_path,
incremental=True, # Only process changed files
file_pattern="**/*.py", # Only Python files
exclude_patterns=["**/test_*.py", "**/migrations/**"],
)
# Display results
print(f"Repository: {result.repo_path}")
print(f"Total files: {result.total_files}")
print(f"Total chunks: {result.total_chunks}")
print(f"Processing time: {result.processing_time:.2f}s")
print(f"Files/second: {result.metadata['stats']['files_per_second']:.1f}")
# Check for errors
if result.errors:
print(f"\nErrors in {len(result.errors)} files:")
for file_path, error in list(result.errors.items())[:5]:
print(f" - {file_path}: {error}")
def process_with_iterator_example():
"""Example: Process repository using iterator for memory efficiency."""
processor = RepoProcessorImpl(max_workers=4)
repo_path = "/path/to/large/repository"
# Process files one at a time to minimize memory usage
total_chunks = 0
total_files = 0
for file_result in processor.process_files_iterator(repo_path):
total_files += 1
total_chunks += len(file_result.chunks)
# Process chunks immediately (e.g., save to database)
for _chunk in file_result.chunks:
# Your processing logic here
pass
# Free memory after processing each file_path
if total_files % 100 == 0:
print(f"Processed {total_files} files, {total_chunks} chunks...")
def git_aware_processing_example():
"""Example: Use Git awareness for smart processing."""
git_processor = GitAwareProcessorImpl()
repo_path = "/path/to/git/repository"
# Get files changed in last 10 commits
changed_files = git_processor.get_changed_files(
repo_path,
since_commit="HEAD~10",
)
print(f"Files changed in last 10 commits: {len(changed_files)}")
# Get file_path history
for file_path in changed_files[:3]:
history = git_processor.get_file_history(
file_path,
repo_path,
limit=5,
)
print(f"\nHistory for {file_path}:")
for commit in history:
print(f" - {commit['date']}: {commit['message']} ({commit['author']})")
# Save state for incremental processing
state = {
"processed_files": changed_files,
"processing_config": {
"chunk_size_threshold": 100,
"languages": ["python", "javascript"],
},
}
git_processor.save_incremental_state(repo_path, state)
def gitignore_pattern_example():
"""Example: Work with gitignore patterns."""
repo_path = Path("/path/to/repository")
# Load all gitignore patterns from repository
matcher = load_gitignore_patterns(repo_path)
# Check specific files
files_to_check = [
"src/main.py",
"build/output.exe",
"node_modules/package.json",
".env.local",
"docs/api.md",
]
for file_path in files_to_check:
if matcher.should_ignore(Path(file_path)):
print(f"IGNORE: {file_path}")
else:
print(f"PROCESS: {file_path}")
# Filter a list of files
all_files = list(repo_path.rglob("*"))
processable_files = matcher.filter_paths(all_files)
print(f"\nTotal files: {len(all_files)}")
print(f"Processable files: {len(processable_files)}")
print(f"Ignored files: {len(all_files) - len(processable_files)}")
if __name__ == "__main__":
print("Repository Processing Examples")
print("=" * 50)
print("\nThese are code examples. Replace paths with actual repositories.")
print("\nKey features demonstrated:")
print("- Parallel processing with memory limits")
print("- Git-aware incremental processing")
print("- Iterator-based processing for large repos")
print("- Gitignore pattern matching")
print("\nSee the code for implementation details.")

Xet Storage Details

Size:
4.91 kB
·
Xet hash:
e5c6fbeb0bd3a323fd18ebaeffe99d2aaf8ed798d0ebc5508a2bc88841a49bd9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.