Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /overlapping_fallback_demo.py
| #!/usr/bin/env python3 | |
| """Demo of overlapping fallback chunker for non-Tree-sitter files.""" | |
| import warnings | |
| from chunker.fallback_overlap import OverlappingFallbackChunker, OverlapStrategy | |
| def demo_log_file_chunking(): | |
| """Demonstrate overlapping chunks on a log file.""" | |
| print("=== Log File Chunking Demo ===\n") | |
| # Sample log content | |
| log_content = """2024-01-15 10:00:00 INFO Starting application server | |
| 2024-01-15 10:00:01 DEBUG Loading configuration from config.yml | |
| 2024-01-15 10:00:02 INFO Configuration loaded successfully | |
| 2024-01-15 10:00:03 ERROR Failed to connect to database: timeout | |
| 2024-01-15 10:00:04 WARN Retrying database connection (attempt 1/3) | |
| 2024-01-15 10:00:05 INFO Database connection established | |
| 2024-01-15 10:00:06 DEBUG Executing startup queries | |
| 2024-01-15 10:00:07 INFO All systems operational | |
| 2024-01-15 10:00:08 DEBUG Starting request handler | |
| 2024-01-15 10:00:09 INFO Server ready on port 8080""" | |
| chunker = OverlappingFallbackChunker() | |
| # Chunk with line-based overlap | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") # Suppress warnings for demo | |
| chunks = chunker.chunk_with_overlap( | |
| log_content, | |
| "server.log", | |
| chunk_size=4, # 4 lines per chunk | |
| overlap_size=2, # 2 lines overlap | |
| strategy=OverlapStrategy.FIXED, | |
| unit="lines", | |
| ) | |
| print(f"Created {len(chunks)} overlapping chunks:\n") | |
| for i, chunk in enumerate(chunks): | |
| lines = chunk.content.strip().split("\n") | |
| print(f"Chunk {i} (lines {chunk.start_line}-{chunk.end_line}):") | |
| for line in lines: | |
| print(f" {line}") | |
| print() | |
| def demo_markdown_chunking(): | |
| """Demonstrate overlapping chunks on markdown content.""" | |
| print("=== Markdown Chunking Demo ===\n") | |
| markdown_content = """# Project README | |
| ## Introduction | |
| This project demonstrates the overlapping fallback chunker. | |
| It's designed for files without Tree-sitter support. | |
| ## Features | |
| - Configurable overlap strategies | |
| - Support for line and character-based chunking | |
| - Natural boundary detection | |
| - Clear warnings when used | |
| ## Usage | |
| The chunker can be used programmatically: | |
| ```python | |
| chunker = OverlappingFallbackChunker() | |
| chunks = chunker.chunk_with_overlap(content, path) | |
| ``` | |
| ## Configuration | |
| You can configure overlap size and strategy. | |
| """ | |
| chunker = OverlappingFallbackChunker() | |
| # Chunk with character-based overlap | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| chunks = chunker.chunk_with_overlap( | |
| markdown_content, | |
| "README.md", | |
| chunk_size=150, # 150 chars per chunk | |
| overlap_size=30, # 20% overlap | |
| strategy=OverlapStrategy.PERCENTAGE, | |
| unit="characters", | |
| ) | |
| print(f"Created {len(chunks)} overlapping chunks:\n") | |
| for i, chunk in enumerate(chunks): | |
| print(f"Chunk {i} ({chunk.byte_start}-{chunk.byte_end} bytes):") | |
| print( | |
| ( | |
| f" Preview: {chunk.content[:60]}..." | |
| if len(chunk.content) > 60 | |
| else f" Content: {chunk.content}" | |
| ), | |
| ) | |
| print() | |
| def demo_error_on_code_file(): | |
| """Demonstrate that code files with Tree-sitter support are rejected.""" | |
| print("=== Error Demo for Code Files ===\n") | |
| chunker = OverlappingFallbackChunker() | |
| try: | |
| # This should fail because Go has Tree-sitter support | |
| chunker.chunk_with_overlap( | |
| "package main\n\nfunc main() {}", | |
| "main.go", | |
| chunk_size=100, | |
| overlap_size=20, | |
| ) | |
| except (FileNotFoundError, OSError, TypeError) as e: | |
| print(f"Expected error occurred: {e}") | |
| print( | |
| "\nThis is correct behavior - overlapping chunks are ONLY for non-Tree-sitter files!", | |
| ) | |
| if __name__ == "__main__": | |
| demo_log_file_chunking() | |
| print("\n" + "=" * 50 + "\n") | |
| demo_markdown_chunking() | |
| print("\n" + "=" * 50 + "\n") | |
| demo_error_on_code_file() | |
Xet Storage Details
- Size:
- 4.09 kB
- Xet hash:
- cfc242912a2a16d5f7fca045609c470f4daac585c0a2294abceaaee0ae4c2cbd
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.