Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /test_overlapping_direct.py
| #!/usr/bin/env python3 | |
| """Direct test of overlapping fallback chunker without full module imports.""" | |
| # Test the overlapping chunker directly | |
| def test_overlapping(): | |
| # Import only the specific modules we need | |
| # Import the base fallback chunker | |
| # Now import our overlapping chunker | |
| from chunker.fallback.overlapping import OverlappingFallbackChunker | |
| from chunker.interfaces.fallback_overlap import OverlapStrategy | |
| print("Successfully imported OverlappingFallbackChunker!") | |
| # Create test content | |
| test_content = """Line 1: This is the first line | |
| Line 2: This is the second line | |
| Line 3: This is the third line | |
| Line 4: This is the fourth line | |
| Line 5: This is the fifth line | |
| Line 6: This is the sixth line | |
| Line 7: This is the seventh line | |
| Line 8: This is the eighth line | |
| Line 9: This is the ninth line | |
| Line 10: This is the tenth line""" | |
| # Create chunker | |
| chunker = OverlappingFallbackChunker() | |
| print("\nCreated chunker instance") | |
| # Test fixed overlap by lines | |
| print("\n=== Testing Fixed Overlap by Lines ===") | |
| chunks = chunker.chunk_with_overlap( | |
| content=test_content, | |
| file_path="test.txt", | |
| chunk_size=3, # 3 lines per chunk | |
| overlap_size=1, # 1 line overlap | |
| strategy=OverlapStrategy.FIXED, | |
| unit="lines", | |
| ) | |
| print(f"Generated {len(chunks)} chunks") | |
| for i, chunk in enumerate(chunks): | |
| print(f"\nChunk {i + 1}:") | |
| print(f" Lines: {chunk.start_line}-{chunk.end_line}") | |
| print(f" Content preview: {chunk.content[:50]}...") | |
| # Test overlap exists | |
| if len(chunks) >= 2: | |
| chunk1_lines = chunks[0].content.splitlines() | |
| chunk2_lines = chunks[1].content.splitlines() | |
| print("\nOverlap check:") | |
| print(f" Last line of chunk 1: {chunk1_lines[-1]}") | |
| print(f" First line of chunk 2: {chunk2_lines[0]}") | |
| print(f" Overlap exists: {chunk1_lines[-1] == chunk2_lines[0]}") | |
| # Test character-based overlap | |
| print("\n=== Testing Fixed Overlap by Characters ===") | |
| chunks_char = chunker.chunk_with_overlap( | |
| content=test_content, | |
| file_path="test.txt", | |
| chunk_size=50, | |
| overlap_size=10, | |
| strategy=OverlapStrategy.FIXED, | |
| unit="characters", | |
| ) | |
| print(f"Generated {len(chunks_char)} chunks") | |
| for i, chunk in enumerate(chunks_char): | |
| print(f"\nChunk {i + 1}:") | |
| print(f" Size: {len(chunk.content)} chars") | |
| print(f" Content: {chunk.content[:30]!r}...") | |
| # Test asymmetric overlap | |
| print("\n=== Testing Asymmetric Overlap ===") | |
| chunks_asym = chunker.chunk_with_asymmetric_overlap( | |
| content=test_content, | |
| file_path="test.txt", | |
| chunk_size=3, | |
| overlap_before=1, | |
| overlap_after=2, | |
| unit="lines", | |
| ) | |
| print(f"Generated {len(chunks_asym)} chunks") | |
| for i, chunk in enumerate(chunks_asym): | |
| lines = chunk.content.splitlines() | |
| print(f"\nChunk {i + 1}: {len(lines)} lines") | |
| # Test natural boundary detection | |
| print("\n=== Testing Natural Boundary Detection ===") | |
| test_text = ( | |
| "This is a sentence. Another sentence here.\n\nNew paragraph starts here." | |
| ) | |
| boundary = chunker.find_natural_overlap_boundary(test_text, 25, 10) | |
| print("Desired position: 25") | |
| print(f"Found boundary at: {boundary}") | |
| print(f"Character at boundary: {test_text[boundary - 1:boundary + 1]!r}") | |
| print("\n✅ All tests completed successfully!") | |
| if __name__ == "__main__": | |
| test_overlapping() | |
Xet Storage Details
- Size:
- 3.54 kB
- Xet hash:
- 6f3b28fe92a8b76ed27d7c9d01202587dce5a291ac5778bd0e3e40662c66836b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.