Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /tests /test_fallback_chunking.py
| """Tests for fallback chunking functionality.""" | |
| import tempfile | |
| from pathlib import Path | |
| import pytest | |
| from chunker.fallback import ( | |
| FallbackWarning, | |
| FileTypeDetector, | |
| LineBasedChunker, | |
| LogChunker, | |
| MarkdownChunker, | |
| ) | |
| from chunker.fallback.detection.file_type import EncodingDetector, FileType | |
| from chunker.fallback.fallback_manager import FallbackManager | |
| from chunker.interfaces.fallback import FallbackReason | |
| class TestFallbackBase: | |
| """Test base fallback functionality.""" | |
| def test_fallback_warning_emitted(cls): | |
| """Test that fallback usage emits warnings.""" | |
| chunker = LineBasedChunker() | |
| chunker.set_fallback_reason(FallbackReason.NO_GRAMMAR) | |
| content = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" | |
| with pytest.warns(FallbackWarning, match="WARNING: Using fallback chunking"): | |
| chunker.chunk_text(content, "test.txt") | |
| def test_line_based_chunking(cls): | |
| """Test basic line-based chunking.""" | |
| chunker = LineBasedChunker(lines_per_chunk=2, overlap=0) | |
| content = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" | |
| chunks = chunker.chunk_by_lines(content, 2, 0) | |
| assert len(chunks) == 3 | |
| assert chunks[0].content == "Line 1\nLine 2\n" | |
| assert chunks[0].start_line == 1 | |
| assert chunks[0].end_line == 2 | |
| assert chunks[1].content == "Line 3\nLine 4\n" | |
| assert chunks[1].start_line == 3 | |
| assert chunks[1].end_line == 4 | |
| assert chunks[2].content == "Line 5" | |
| assert chunks[2].start_line == 5 | |
| assert chunks[2].end_line == 5 | |
| def test_line_based_with_overlap(cls): | |
| """Test line-based chunking with overlap.""" | |
| chunker = LineBasedChunker(lines_per_chunk=3, overlap=1) | |
| content = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" | |
| chunks = chunker.chunk_by_lines(content, 3, 1) | |
| assert len(chunks) == 2 | |
| assert "Line 1" in chunks[0].content | |
| assert "Line 3" in chunks[0].content | |
| assert "Line 3" in chunks[1].content | |
| assert "Line 5" in chunks[1].content | |
| def test_delimiter_based_chunking(cls): | |
| """Test delimiter-based chunking.""" | |
| chunker = LineBasedChunker() | |
| content = "Section 1\n---\nSection 2\n---\nSection 3" | |
| chunks = chunker.chunk_by_delimiter(content, "---", include_delimiter=False) | |
| assert len(chunks) == 3 | |
| assert chunks[0].content.strip() == "Section 1" | |
| assert chunks[1].content.strip() == "Section 2" | |
| assert chunks[2].content.strip() == "Section 3" | |
| class TestFileTypeDetection: | |
| """Test file type detection.""" | |
| def test_extension_detection(cls): | |
| """Test detection by file extension.""" | |
| detector = FileTypeDetector() | |
| assert detector.detect_file_type("test.log") == FileType.LOG | |
| assert detector.detect_file_type("README.md") == FileType.MARKDOWN | |
| assert detector.detect_file_type("data.csv") == FileType.CSV | |
| assert detector.detect_file_type("config.yaml") == FileType.YAML | |
| assert detector.detect_file_type("notes.txt") == FileType.TEXT | |
| def test_content_detection(cls): | |
| """Test detection by content patterns.""" | |
| detector = FileTypeDetector() | |
| with tempfile.NamedTemporaryFile( | |
| encoding="utf-8", | |
| mode="w", | |
| suffix=".unknown", | |
| delete=False, | |
| ) as f: | |
| f.write("2024-01-15 10:30:45 ERROR Something went wrong\n") | |
| f.write("2024-01-15 10:30:46 INFO Process started\n") | |
| f.flush() | |
| temp_path = Path(f.name) | |
| try: | |
| file_type = detector.detect_file_type(str(temp_path)) | |
| assert file_type == FileType.LOG | |
| finally: | |
| temp_path.unlink(missing_ok=True) | |
| def test_encoding_detection(cls): | |
| """Test encoding detection.""" | |
| with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as f: | |
| f.write("Hello, world! 你好世界") | |
| f.flush() | |
| temp_path = Path(f.name) | |
| try: | |
| encoding, _confidence = EncodingDetector.detect_encoding(str(temp_path)) | |
| assert encoding.lower() in {"utf-8", "utf8"} | |
| content, _used_encoding = EncodingDetector.read_with_encoding( | |
| str(temp_path) | |
| ) | |
| assert "Hello, world!" in content | |
| assert "你好世界" in content | |
| finally: | |
| temp_path.unlink(missing_ok=True) | |
| class TestLogChunker: | |
| """Test log file chunking.""" | |
| def test_timestamp_detection(cls): | |
| """Test timestamp pattern detection.""" | |
| chunker = LogChunker() | |
| log_content = """2024-01-15T10:30:45.123Z INFO Starting application | |
| 2024-01-15T10:30:46.456Z DEBUG Configuration loaded | |
| 2024-01-15T10:31:00.789Z ERROR Connection failed""" | |
| format_info = chunker.detect_log_format(log_content) | |
| assert format_info["has_timestamps"] is True | |
| assert format_info["has_levels"] is True | |
| def test_chunk_by_timestamp(cls): | |
| """Test chunking logs by time window.""" | |
| chunker = LogChunker() | |
| chunker.file_path = "test.log" | |
| log_content = """2024-01-15 10:30:00 INFO Start | |
| 2024-01-15 10:30:30 DEBUG Processing | |
| 2024-01-15 10:31:00 INFO Checkpoint | |
| 2024-01-15 10:31:30 DEBUG More processing | |
| 2024-01-15 10:32:00 INFO Complete""" | |
| chunks = chunker.chunk_by_timestamp(log_content, 60) | |
| assert len(chunks) >= 2 | |
| assert "10:30:00" in chunks[0].content | |
| assert "10:30:30" in chunks[0].content | |
| def test_chunk_by_severity(cls): | |
| """Test chunking logs by severity level.""" | |
| chunker = LogChunker() | |
| chunker.file_path = "test.log" | |
| log_content = """INFO Starting process | |
| INFO Configuration loaded | |
| ERROR Failed to connect | |
| ERROR Retry failed | |
| WARN Using fallback | |
| INFO Process complete""" | |
| chunks = chunker.chunk_by_severity(log_content, group_consecutive=True) | |
| assert len(chunks) == 4 | |
| assert "Starting process" in chunks[0].content | |
| assert "Configuration loaded" in chunks[0].content | |
| assert "Failed to connect" in chunks[1].content | |
| assert "Retry failed" in chunks[1].content | |
| class TestMarkdownChunker: | |
| """Test markdown file chunking.""" | |
| def test_chunk_by_headers(cls): | |
| """Test chunking markdown by headers.""" | |
| chunker = MarkdownChunker() | |
| chunker.file_path = "test.md" | |
| md_content = """# Title | |
| Introduction paragraph. | |
| ## Section 1 | |
| Content for section 1. | |
| ### Subsection 1.1 | |
| Detailed content. | |
| ## Section 2 | |
| Content for section 2.""" | |
| chunks = chunker.chunk_by_headers(md_content, max_level=2) | |
| assert len(chunks) >= 3 | |
| section1_chunk = next(c for c in chunks if "Section 1" in c.content) | |
| assert "Subsection 1.1" in section1_chunk.content | |
| def test_chunk_by_sections(cls): | |
| """Test chunking markdown by logical sections.""" | |
| chunker = MarkdownChunker() | |
| chunker.file_path = "test.md" | |
| md_content = """# Title | |
| Introduction paragraph. | |
| ```python | |
| def hello(): | |
| print("Hello") | |
| ``` | |
| Some more text. | |
| - Item 1 | |
| - Item 2 | |
| - Item 3 | |
| Final paragraph.""" | |
| chunks = chunker.chunk_by_sections( | |
| md_content, | |
| include_code_blocks=True, | |
| ) | |
| assert len(chunks) >= 4 | |
| code_chunk = next((c for c in chunks if "code_block" in c.node_type), None) | |
| assert code_chunk is not None | |
| assert "def hello():" in code_chunk.content | |
| def test_extract_code_blocks(cls): | |
| """Test extracting code blocks from markdown.""" | |
| chunker = MarkdownChunker() | |
| chunker.file_path = "test.md" | |
| md_content = """# Example | |
| Here's some Python code: | |
| ```python | |
| def greet(name): | |
| return f"Hello, {name}!" | |
| ``` | |
| And some JavaScript: | |
| ```javascript | |
| function greet(name) { | |
| return `Hello, ${name}!`; | |
| } | |
| ``` | |
| """ | |
| chunks = chunker.extract_code_blocks(md_content) | |
| assert len(chunks) == 2 | |
| python_chunk = next(c for c in chunks if c.language == "python") | |
| assert "def greet" in python_chunk.content | |
| js_chunk = next(c for c in chunks if c.language == "javascript") | |
| assert "function greet" in js_chunk.content | |
| class TestFallbackManager: | |
| """Test fallback manager coordination.""" | |
| def test_manager_file_detection(cls): | |
| """Test manager's file type detection and chunking.""" | |
| manager = FallbackManager() | |
| with tempfile.NamedTemporaryFile( | |
| encoding="utf-8", | |
| mode="w", | |
| suffix=".log", | |
| delete=False, | |
| ) as f: | |
| f.write("2024-01-15 10:30:00 INFO Test log entry\n") | |
| f.write("2024-01-15 10:30:01 ERROR Something failed\n") | |
| f.flush() | |
| temp_path = Path(f.name) | |
| try: | |
| assert manager.can_chunk(str(temp_path)) is True | |
| with pytest.warns(FallbackWarning, match="Using fallback chunking"): | |
| chunks = manager.chunk_file(str(temp_path)) | |
| assert len(chunks) > 0 | |
| assert any("Test log entry" in chunk.content for chunk in chunks) | |
| finally: | |
| temp_path.unlink(missing_ok=True) | |
| def test_manager_fallback_info(cls): | |
| """Test getting fallback information.""" | |
| manager = FallbackManager() | |
| with tempfile.NamedTemporaryFile( | |
| encoding="utf-8", | |
| mode="w", | |
| suffix=".csv", | |
| delete=False, | |
| ) as f: | |
| f.write("name,age,city\n") | |
| f.write("Alice,30,NYC\n") | |
| f.flush() | |
| temp_path = Path(f.name) | |
| try: | |
| info = manager.get_fallback_info(str(temp_path)) | |
| assert info["file_type"] == "csv" | |
| assert info["can_chunk"] is True | |
| assert info["should_use_fallback"] is True | |
| assert info["fallback_reason"] == FallbackReason.NO_GRAMMAR.value | |
| finally: | |
| temp_path.unlink(missing_ok=True) | |
| def test_csv_chunking(cls): | |
| """Test CSV-specific chunking.""" | |
| manager = FallbackManager() | |
| with tempfile.NamedTemporaryFile( | |
| encoding="utf-8", | |
| mode="w", | |
| # newline="" disables the platform newline translation so the file | |
| # on disk is LF (not CRLF on Windows). The chunker's byte offsets are | |
| # relative to the newline-normalized content it reads, so the raw | |
| # file bytes must also be LF for the slice-back assertion below to | |
| # hold cross-platform. | |
| newline="", | |
| suffix=".csv", | |
| delete=False, | |
| ) as f: | |
| f.write("id,name,score\n") | |
| for i in range(60): | |
| f.write(f"{i},User{i},{i * 10}\n") | |
| f.flush() | |
| temp_path = Path(f.name) | |
| try: | |
| with pytest.warns(FallbackWarning, match="Using fallback chunking"): | |
| chunks = manager.chunk_file(str(temp_path)) | |
| assert len(chunks) > 0 | |
| assert ( | |
| len( | |
| chunks, | |
| ) | |
| > 1 | |
| ), "Need multiple chunks to test header inclusion" | |
| assert chunks[0].start_line == 2 | |
| # The CSV header is preserved OUT-OF-BAND in metadata (not prepended | |
| # into content), so every chunk's content is a contiguous data-row | |
| # slice that slices back to its byte span (README universal | |
| # slice-back contract, incl. fallback chunks). Prepending the header | |
| # made content non-contiguous and broke that. | |
| source_bytes = temp_path.read_bytes() | |
| for i, chunk in enumerate(chunks): | |
| assert ( | |
| "id,name,score" not in chunk.content | |
| ), f"Chunk {i} must not inline the header into content" | |
| assert ( | |
| chunk.metadata.get("csv_header") == "id,name,score" | |
| ), f"Chunk {i} lost the CSV header from metadata" | |
| sliced = source_bytes[chunk.byte_start : chunk.byte_end].decode("utf-8") | |
| assert ( | |
| sliced == chunk.content | |
| ), f"Chunk {i} byte span does not slice back to content" | |
| finally: | |
| temp_path.unlink(missing_ok=True) | |
Xet Storage Details
- Size:
- 12.6 kB
- Xet hash:
- 9bd37a870595f8adc1132c70eb2db43993736b3c731b16c6b321cd72a5188481
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.