Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /performance_demo.py
| """Demonstration of performance optimization features.""" | |
| import tempfile | |
| import time | |
| from pathlib import Path | |
| from chunker.performance.enhanced_chunker import EnhancedChunker | |
| from chunker.performance.optimization.batch import BatchProcessor | |
| from chunker.performance.optimization.memory_pool import MemoryPool | |
| from chunker.performance.optimization.monitor import PerformanceMonitor | |
| def demo_basic_caching(): | |
| """Demonstrate basic caching functionality.""" | |
| print("\n=== Basic Caching Demo ===") | |
| # Create enhanced chunker | |
| chunker = EnhancedChunker() | |
| # Parse a file (cold cache) | |
| test_file = Path(__file__) | |
| print(f"\nParsing {test_file.name} with cold cache...") | |
| start = time.perf_counter() | |
| chunks1 = chunker.chunk_file(test_file, "python") | |
| cold_time = time.perf_counter() - start | |
| print(f"Cold cache: {cold_time * 1000:.2f}ms, {len(chunks1)} chunks") | |
| # Parse again (warm cache) | |
| print(f"\nParsing {test_file.name} with warm cache...") | |
| start = time.perf_counter() | |
| chunks2 = chunker.chunk_file(test_file, "python") | |
| warm_time = time.perf_counter() - start | |
| print(f"Warm cache: {warm_time * 1000:.2f}ms, {len(chunks2)} chunks") | |
| # Show speedup | |
| speedup = cold_time / warm_time | |
| print(f"\nSpeedup: {speedup:.2f}x") | |
| # Display cache statistics | |
| stats = chunker.get_stats() | |
| cache_stats = stats["cache"] | |
| print("\nCache statistics:") | |
| print(f" Hit rate: {cache_stats['overall_hit_rate']:.2%}") | |
| print(f" Total size: {cache_stats['total_size']} entries") | |
| print(f" Memory usage: {cache_stats['total_memory_bytes'] / 1024:.2f} KB") | |
| def demo_incremental_parsing(): | |
| """Demonstrate incremental parsing.""" | |
| print("\n\n=== Incremental Parsing Demo ===") | |
| # Create a test file | |
| test_content = ''' | |
| def function_one(): | |
| """First function.""" | |
| return 1 | |
| def function_two(): | |
| """Second function.""" | |
| return 2 | |
| class TestClass: | |
| """Test class.""" | |
| def method(self): | |
| return 3 | |
| ''' | |
| with tempfile.NamedTemporaryFile( | |
| encoding="utf-8", | |
| mode="w", | |
| suffix=".py", | |
| delete=False, | |
| ) as f: | |
| f.write(test_content) | |
| test_file = Path(f.name) | |
| try: | |
| chunker = EnhancedChunker(enable_incremental=True) | |
| # Initial parse | |
| print("\nInitial parse...") | |
| chunks1 = chunker.chunk_file(test_file, "python") | |
| print(f"Found {len(chunks1)} chunks") | |
| # Make a small change | |
| new_content = test_content.replace("return 1", "return 100") | |
| test_file.write_text(new_content, encoding="utf-8") | |
| # Incremental parse | |
| print("\nIncremental parse after small change...") | |
| start = time.perf_counter() | |
| chunker.chunk_file_incremental(test_file, "python") | |
| incr_time = time.perf_counter() - start | |
| print(f"Incremental parse: {incr_time * 1000:.2f}ms") | |
| # Compare with full reparse | |
| chunker.invalidate_file(test_file) # Clear cache | |
| start = time.perf_counter() | |
| chunker.chunk_file(test_file, "python", force_reparse=True) | |
| full_time = time.perf_counter() - start | |
| print(f"Full reparse: {full_time * 1000:.2f}ms") | |
| if incr_time < full_time: | |
| print(f"\nIncremental parsing was {full_time / incr_time:.2f}x faster!") | |
| finally: | |
| test_file.unlink() | |
| def demo_batch_processing(): | |
| """Demonstrate batch processing.""" | |
| print("\n\n=== Batch Processing Demo ===") | |
| # Find Python files in current directory | |
| current_dir = Path(__file__).parent | |
| py_files = list(current_dir.glob("*.py"))[:5] # Limit to 5 files | |
| if not py_files: | |
| print("No Python files found in current directory") | |
| return | |
| print(f"\nFound {len(py_files)} Python files to process") | |
| # Create batch processor with monitoring | |
| monitor = PerformanceMonitor() | |
| processor = BatchProcessor( | |
| performance_monitor=monitor, | |
| max_workers=2, | |
| ) | |
| # Add files with size-based priority | |
| for file_path in py_files: | |
| size = file_path.stat().st_size | |
| processor.add_file(str(file_path), priority=size) | |
| print(f" Added {file_path.name} (priority: {size})") | |
| # Process batch | |
| print("\nProcessing batch...") | |
| start = time.perf_counter() | |
| results = processor.process_batch(batch_size=len(py_files), parallel=True) | |
| elapsed = time.perf_counter() - start | |
| # Show results | |
| total_chunks = sum(len(chunks) for chunks in results.values()) | |
| print(f"\nProcessed {len(results)} files in {elapsed:.2f}s") | |
| print(f"Total chunks: {total_chunks}") | |
| print(f"Files/second: {len(results) / elapsed:.1f}") | |
| # Show performance metrics | |
| print("\nPerformance metrics:") | |
| metrics = monitor.get_metrics() | |
| for name, stats in sorted(metrics.items()): | |
| if stats["count"] > 0: | |
| print(f" {name}: {stats['mean']:.2f} (avg of {stats['count']})") | |
| def demo_memory_pooling(): | |
| """Demonstrate memory pooling benefits.""" | |
| print("\n\n=== Memory Pooling Demo ===") | |
| pool = MemoryPool(max_pool_size=5) | |
| # Warm up pool | |
| print("\nWarming up parser pool...") | |
| pool.warm_up("parser:python", 3) | |
| # Acquire and release parsers | |
| print("\nAcquiring and releasing parsers...") | |
| parsers_used = [] | |
| for i in range(5): | |
| parser = pool.acquire_parser("python") | |
| parsers_used.append(parser) | |
| print(f" Acquired parser {i + 1}") | |
| # Release them back | |
| for i, parser in enumerate(parsers_used): | |
| pool.release_parser(parser, "python") | |
| print(f" Released parser {i + 1}") | |
| # Show pool statistics | |
| stats = pool.get_stats() | |
| if "parser:python" in stats: | |
| p_stats = stats["parser:python"] | |
| print("\nPool statistics:") | |
| print(f" Created: {p_stats['created']} parsers") | |
| print(f" Acquired: {p_stats['acquired']} times") | |
| print(f" Released: {p_stats['released']} times") | |
| print(f" Currently pooled: {p_stats['pooled']}") | |
| reuse_rate = ( | |
| p_stats["acquired"] / p_stats["created"] if p_stats["created"] > 0 else 0 | |
| ) | |
| print(f" Reuse rate: {reuse_rate:.2f}x") | |
| def demo_performance_monitoring(): | |
| """Demonstrate performance monitoring.""" | |
| print("\n\n=== Performance Monitoring Demo ===") | |
| monitor = PerformanceMonitor() | |
| # Time various operations | |
| operations = [ | |
| ("fast_operation", 0.01), | |
| ("medium_operation", 0.05), | |
| ("slow_operation", 0.1), | |
| ] | |
| print("\nTiming operations...") | |
| for op_name, sleep_time in operations: | |
| with monitor.measure(op_name): | |
| time.sleep(sleep_time) # Simulate work | |
| print(f" Executed {op_name}") | |
| # Record some metrics | |
| print("\nRecording metrics...") | |
| for i in range(10): | |
| monitor.record_metric("file_size", 1000 + i * 500) | |
| monitor.record_metric("chunk_count", 5 + i) | |
| # Display summary | |
| print("\n" + monitor.get_summary()) | |
| def main(): | |
| """Run all demonstrations.""" | |
| print("Tree-sitter Chunker Performance Optimization Demo") | |
| print("=" * 50) | |
| demos = [ | |
| demo_basic_caching, | |
| demo_incremental_parsing, | |
| demo_batch_processing, | |
| demo_memory_pooling, | |
| demo_performance_monitoring, | |
| ] | |
| for demo in demos: | |
| try: | |
| demo() | |
| except (TypeError, ValueError) as e: | |
| print(f"\nError in {demo.__name__}: {e}") | |
| print("\n\nDemo complete!") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 7.52 kB
- Xet hash:
- c6cedfbdec6b1c0e77ddc0ebc811e03df4effcad1decb3aece2834be3db18f08
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.