Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /optimize_chunks.py
| """Example of chunk optimization for different use cases.""" | |
| from pathlib import Path | |
| from chunker import ( | |
| ChunkBoundaryAnalyzer, | |
| ChunkOptimizer, | |
| OptimizationConfig, | |
| OptimizationStrategy, | |
| chunk_file, | |
| ) | |
| def demonstrate_optimization(): | |
| """Demonstrate chunk optimization features.""" | |
| # 1. Get initial chunks from a file | |
| print("1. Getting initial chunks from a Python file...") | |
| chunks = chunk_file("examples/sample_code.py", language="python") | |
| print(f" Original chunks: {len(chunks)}") | |
| # 2. Create optimizer with default config | |
| optimizer = ChunkOptimizer() | |
| # 3. Optimize for different LLM models | |
| print("\n2. Optimizing for GPT-4 (8k context)...") | |
| _optimized_gpt4, metrics_gpt4 = optimizer.optimize_for_llm( | |
| chunks, | |
| model="gpt-4", | |
| max_tokens=2000, # Leave room for prompts | |
| strategy=OptimizationStrategy.BALANCED, | |
| ) | |
| print(f" Optimized chunks: {metrics_gpt4.optimized_count}") | |
| print(f" Average tokens: {metrics_gpt4.avg_tokens_after:.1f}") | |
| print(f" Coherence score: {metrics_gpt4.coherence_score:.2f}") | |
| print(f" Token efficiency: {metrics_gpt4.token_efficiency:.2%}") | |
| # 4. Optimize for Claude (100k context) | |
| print("\n3. Optimizing for Claude (100k context)...") | |
| _optimized_claude, metrics_claude = optimizer.optimize_for_llm( | |
| chunks, | |
| model="claude", | |
| max_tokens=8000, # Can use larger chunks | |
| strategy=OptimizationStrategy.AGGRESSIVE, | |
| ) | |
| print(f" Optimized chunks: {metrics_claude.optimized_count}") | |
| print(f" Average tokens: {metrics_claude.avg_tokens_after:.1f}") | |
| # 5. Optimize for embeddings | |
| print("\n4. Optimizing for embeddings...") | |
| embedding_chunks = optimizer.optimize_for_embedding( | |
| chunks, | |
| embedding_model="text-embedding-ada-002", | |
| max_tokens=512, | |
| ) | |
| print(f" Embedding chunks: {len(embedding_chunks)}") | |
| # 6. Custom configuration | |
| print("\n5. Using custom configuration...") | |
| config = OptimizationConfig() | |
| config.min_chunk_tokens = 100 | |
| config.max_chunk_tokens = 1500 | |
| config.target_chunk_tokens = 750 | |
| config.merge_threshold = 0.8 | |
| custom_optimizer = ChunkOptimizer(config) | |
| # Rebalance chunks for uniform sizing | |
| rebalanced = custom_optimizer.rebalance_chunks( | |
| chunks, | |
| target_tokens=750, | |
| variance=0.2, | |
| ) | |
| print(f" Rebalanced chunks: {len(rebalanced)}") | |
| # 7. Preserve structure strategy | |
| print("\n6. Using PRESERVE_STRUCTURE strategy...") | |
| _preserved, metrics_preserved = optimizer.optimize_for_llm( | |
| chunks, | |
| model="gpt-4", | |
| max_tokens=1000, | |
| strategy=OptimizationStrategy.PRESERVE_STRUCTURE, | |
| ) | |
| print(f" Preserved chunks: {metrics_preserved.optimized_count}") | |
| print( | |
| f" Structure maintained: {metrics_preserved.optimized_count == len(chunks)}", | |
| ) | |
| # 8. Show chunk boundaries analysis | |
| print("\n7. Analyzing chunk boundaries...") | |
| analyzer = ChunkBoundaryAnalyzer() | |
| # Find merge suggestions | |
| merge_suggestions = analyzer.suggest_merge_points(chunks[:5]) # First 5 chunks | |
| print(f" Merge suggestions: {len(merge_suggestions)}") | |
| for _i, (idx1, idx2, score) in enumerate(merge_suggestions[:3]): | |
| print(f" - Merge chunks {idx1} and {idx2} (score: {score:.2f})") | |
| def create_sample_code(): | |
| """Create a sample code file for demonstration.""" | |
| sample_code = '''"""Sample Python module for chunk optimization demo.""" | |
| import os | |
| import sys | |
| from typing import List, Dict, Optional | |
| class DataProcessor: | |
| """Processes data with various methods.""" | |
| def __init__(self, config: Dict[str, any]): | |
| self.config = config | |
| self.data = [] | |
| self.results = {} | |
| def load_data(self, filepath: str) -> bool: | |
| """Load data from file.""" | |
| try: | |
| with Path(filepath).open('r', ) as f: | |
| self.data = f.readlines() | |
| return True | |
| except (FileNotFoundError, IOError, IndexError) as e: | |
| print(f"Error loading data: {e}") | |
| return False | |
| def process_batch(self, batch_size: int = 100) -> List[Dict]: | |
| """Process data in batches.""" | |
| results = [] | |
| for i in range(0, len(self.data), batch_size): | |
| batch = self.data[i:i + batch_size] | |
| processed = self._process_single_batch(batch) | |
| results.extend(processed) | |
| return results | |
| def _process_single_batch(self, batch: List[str]) -> List[Dict]: | |
| """Process a single batch of data.""" | |
| processed = [] | |
| for item in batch: | |
| # Complex processing logic here | |
| result = { | |
| 'original': item, | |
| 'processed': item.strip().upper(), | |
| 'length': len(item) | |
| } | |
| processed.append(result) | |
| return processed | |
| def save_results(self, output_path: str) -> bool: | |
| """Save processing results.""" | |
| try: | |
| with Path(output_path).open('w', ) as f: | |
| for result in self.results: | |
| f.write(str(result) + '\\n') | |
| return True | |
| except (FileNotFoundError, IOError, IndexError) as e: | |
| print(f"Error saving results: {e}") | |
| return False | |
| def analyze_text(text: str) -> Dict[str, int]: | |
| """Analyze text and return statistics.""" | |
| words = text.split() | |
| lines = text.split('\\n') | |
| stats = { | |
| 'characters': len(text), | |
| 'words': len(words), | |
| 'lines': len(lines), | |
| 'avg_word_length': sum(len(w) for w in words) / len(words) if words else 0 | |
| } | |
| return stats | |
| def main(): | |
| """Main entry point.""" | |
| processor = DataProcessor({'debug': True}) | |
| if processor.load_data('input.txt'): | |
| results = processor.process_batch(50) | |
| processor.results = results | |
| processor.save_results('output.txt') | |
| # Analyze results | |
| for result in results[:10]: | |
| stats = analyze_text(result['processed']) | |
| print(f"Stats: {stats}") | |
| if __name__ == '__main__': | |
| main() | |
| ''' | |
| with Path("examples/sample_code.py").open( | |
| "w", | |
| encoding="utf-8", | |
| ) as f: | |
| f.write(sample_code) | |
| print("Created examples/sample_code.py") | |
| if __name__ == "__main__": | |
| # Create sample file if it doesn't exist | |
| if not Path("examples/sample_code.py").exists(): | |
| Path("examples").mkdir(parents=True, exist_ok=True) | |
| create_sample_code() | |
| demonstrate_optimization() | |
Xet Storage Details
- Size:
- 6.57 kB
- Xet hash:
- d481cf5eb7a36128af693cefe47d1fc8d9b65adbe81c051539e76bd49cb99124
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.