Buckets:

download
raw
12.6 kB
"""Tests for fallback chunking functionality."""
import tempfile
from pathlib import Path
import pytest
from chunker.fallback import (
FallbackWarning,
FileTypeDetector,
LineBasedChunker,
LogChunker,
MarkdownChunker,
)
from chunker.fallback.detection.file_type import EncodingDetector, FileType
from chunker.fallback.fallback_manager import FallbackManager
from chunker.interfaces.fallback import FallbackReason
class TestFallbackBase:
"""Test base fallback functionality."""
@classmethod
def test_fallback_warning_emitted(cls):
"""Test that fallback usage emits warnings."""
chunker = LineBasedChunker()
chunker.set_fallback_reason(FallbackReason.NO_GRAMMAR)
content = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5"
with pytest.warns(FallbackWarning, match="WARNING: Using fallback chunking"):
chunker.chunk_text(content, "test.txt")
@classmethod
def test_line_based_chunking(cls):
"""Test basic line-based chunking."""
chunker = LineBasedChunker(lines_per_chunk=2, overlap=0)
content = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5"
chunks = chunker.chunk_by_lines(content, 2, 0)
assert len(chunks) == 3
assert chunks[0].content == "Line 1\nLine 2\n"
assert chunks[0].start_line == 1
assert chunks[0].end_line == 2
assert chunks[1].content == "Line 3\nLine 4\n"
assert chunks[1].start_line == 3
assert chunks[1].end_line == 4
assert chunks[2].content == "Line 5"
assert chunks[2].start_line == 5
assert chunks[2].end_line == 5
@classmethod
def test_line_based_with_overlap(cls):
"""Test line-based chunking with overlap."""
chunker = LineBasedChunker(lines_per_chunk=3, overlap=1)
content = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5"
chunks = chunker.chunk_by_lines(content, 3, 1)
assert len(chunks) == 2
assert "Line 1" in chunks[0].content
assert "Line 3" in chunks[0].content
assert "Line 3" in chunks[1].content
assert "Line 5" in chunks[1].content
@classmethod
def test_delimiter_based_chunking(cls):
"""Test delimiter-based chunking."""
chunker = LineBasedChunker()
content = "Section 1\n---\nSection 2\n---\nSection 3"
chunks = chunker.chunk_by_delimiter(content, "---", include_delimiter=False)
assert len(chunks) == 3
assert chunks[0].content.strip() == "Section 1"
assert chunks[1].content.strip() == "Section 2"
assert chunks[2].content.strip() == "Section 3"
class TestFileTypeDetection:
"""Test file type detection."""
@classmethod
def test_extension_detection(cls):
"""Test detection by file extension."""
detector = FileTypeDetector()
assert detector.detect_file_type("test.log") == FileType.LOG
assert detector.detect_file_type("README.md") == FileType.MARKDOWN
assert detector.detect_file_type("data.csv") == FileType.CSV
assert detector.detect_file_type("config.yaml") == FileType.YAML
assert detector.detect_file_type("notes.txt") == FileType.TEXT
@classmethod
def test_content_detection(cls):
"""Test detection by content patterns."""
detector = FileTypeDetector()
with tempfile.NamedTemporaryFile(
encoding="utf-8",
mode="w",
suffix=".unknown",
delete=False,
) as f:
f.write("2024-01-15 10:30:45 ERROR Something went wrong\n")
f.write("2024-01-15 10:30:46 INFO Process started\n")
f.flush()
temp_path = Path(f.name)
try:
file_type = detector.detect_file_type(str(temp_path))
assert file_type == FileType.LOG
finally:
temp_path.unlink(missing_ok=True)
@classmethod
def test_encoding_detection(cls):
"""Test encoding detection."""
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as f:
f.write("Hello, world! 你好世界")
f.flush()
temp_path = Path(f.name)
try:
encoding, _confidence = EncodingDetector.detect_encoding(str(temp_path))
assert encoding.lower() in {"utf-8", "utf8"}
content, _used_encoding = EncodingDetector.read_with_encoding(
str(temp_path)
)
assert "Hello, world!" in content
assert "你好世界" in content
finally:
temp_path.unlink(missing_ok=True)
class TestLogChunker:
"""Test log file chunking."""
@classmethod
def test_timestamp_detection(cls):
"""Test timestamp pattern detection."""
chunker = LogChunker()
log_content = """2024-01-15T10:30:45.123Z INFO Starting application
2024-01-15T10:30:46.456Z DEBUG Configuration loaded
2024-01-15T10:31:00.789Z ERROR Connection failed"""
format_info = chunker.detect_log_format(log_content)
assert format_info["has_timestamps"] is True
assert format_info["has_levels"] is True
@classmethod
def test_chunk_by_timestamp(cls):
"""Test chunking logs by time window."""
chunker = LogChunker()
chunker.file_path = "test.log"
log_content = """2024-01-15 10:30:00 INFO Start
2024-01-15 10:30:30 DEBUG Processing
2024-01-15 10:31:00 INFO Checkpoint
2024-01-15 10:31:30 DEBUG More processing
2024-01-15 10:32:00 INFO Complete"""
chunks = chunker.chunk_by_timestamp(log_content, 60)
assert len(chunks) >= 2
assert "10:30:00" in chunks[0].content
assert "10:30:30" in chunks[0].content
@classmethod
def test_chunk_by_severity(cls):
"""Test chunking logs by severity level."""
chunker = LogChunker()
chunker.file_path = "test.log"
log_content = """INFO Starting process
INFO Configuration loaded
ERROR Failed to connect
ERROR Retry failed
WARN Using fallback
INFO Process complete"""
chunks = chunker.chunk_by_severity(log_content, group_consecutive=True)
assert len(chunks) == 4
assert "Starting process" in chunks[0].content
assert "Configuration loaded" in chunks[0].content
assert "Failed to connect" in chunks[1].content
assert "Retry failed" in chunks[1].content
class TestMarkdownChunker:
"""Test markdown file chunking."""
@classmethod
def test_chunk_by_headers(cls):
"""Test chunking markdown by headers."""
chunker = MarkdownChunker()
chunker.file_path = "test.md"
md_content = """# Title
Introduction paragraph.
## Section 1
Content for section 1.
### Subsection 1.1
Detailed content.
## Section 2
Content for section 2."""
chunks = chunker.chunk_by_headers(md_content, max_level=2)
assert len(chunks) >= 3
section1_chunk = next(c for c in chunks if "Section 1" in c.content)
assert "Subsection 1.1" in section1_chunk.content
@classmethod
def test_chunk_by_sections(cls):
"""Test chunking markdown by logical sections."""
chunker = MarkdownChunker()
chunker.file_path = "test.md"
md_content = """# Title
Introduction paragraph.
```python
def hello():
print("Hello")
```
Some more text.
- Item 1
- Item 2
- Item 3
Final paragraph."""
chunks = chunker.chunk_by_sections(
md_content,
include_code_blocks=True,
)
assert len(chunks) >= 4
code_chunk = next((c for c in chunks if "code_block" in c.node_type), None)
assert code_chunk is not None
assert "def hello():" in code_chunk.content
@classmethod
def test_extract_code_blocks(cls):
"""Test extracting code blocks from markdown."""
chunker = MarkdownChunker()
chunker.file_path = "test.md"
md_content = """# Example
Here's some Python code:
```python
def greet(name):
return f"Hello, {name}!"
```
And some JavaScript:
```javascript
function greet(name) {
return `Hello, ${name}!`;
}
```
"""
chunks = chunker.extract_code_blocks(md_content)
assert len(chunks) == 2
python_chunk = next(c for c in chunks if c.language == "python")
assert "def greet" in python_chunk.content
js_chunk = next(c for c in chunks if c.language == "javascript")
assert "function greet" in js_chunk.content
class TestFallbackManager:
"""Test fallback manager coordination."""
@classmethod
def test_manager_file_detection(cls):
"""Test manager's file type detection and chunking."""
manager = FallbackManager()
with tempfile.NamedTemporaryFile(
encoding="utf-8",
mode="w",
suffix=".log",
delete=False,
) as f:
f.write("2024-01-15 10:30:00 INFO Test log entry\n")
f.write("2024-01-15 10:30:01 ERROR Something failed\n")
f.flush()
temp_path = Path(f.name)
try:
assert manager.can_chunk(str(temp_path)) is True
with pytest.warns(FallbackWarning, match="Using fallback chunking"):
chunks = manager.chunk_file(str(temp_path))
assert len(chunks) > 0
assert any("Test log entry" in chunk.content for chunk in chunks)
finally:
temp_path.unlink(missing_ok=True)
@classmethod
def test_manager_fallback_info(cls):
"""Test getting fallback information."""
manager = FallbackManager()
with tempfile.NamedTemporaryFile(
encoding="utf-8",
mode="w",
suffix=".csv",
delete=False,
) as f:
f.write("name,age,city\n")
f.write("Alice,30,NYC\n")
f.flush()
temp_path = Path(f.name)
try:
info = manager.get_fallback_info(str(temp_path))
assert info["file_type"] == "csv"
assert info["can_chunk"] is True
assert info["should_use_fallback"] is True
assert info["fallback_reason"] == FallbackReason.NO_GRAMMAR.value
finally:
temp_path.unlink(missing_ok=True)
@classmethod
def test_csv_chunking(cls):
"""Test CSV-specific chunking."""
manager = FallbackManager()
with tempfile.NamedTemporaryFile(
encoding="utf-8",
mode="w",
# newline="" disables the platform newline translation so the file
# on disk is LF (not CRLF on Windows). The chunker's byte offsets are
# relative to the newline-normalized content it reads, so the raw
# file bytes must also be LF for the slice-back assertion below to
# hold cross-platform.
newline="",
suffix=".csv",
delete=False,
) as f:
f.write("id,name,score\n")
for i in range(60):
f.write(f"{i},User{i},{i * 10}\n")
f.flush()
temp_path = Path(f.name)
try:
with pytest.warns(FallbackWarning, match="Using fallback chunking"):
chunks = manager.chunk_file(str(temp_path))
assert len(chunks) > 0
assert (
len(
chunks,
)
> 1
), "Need multiple chunks to test header inclusion"
assert chunks[0].start_line == 2
# The CSV header is preserved OUT-OF-BAND in metadata (not prepended
# into content), so every chunk's content is a contiguous data-row
# slice that slices back to its byte span (README universal
# slice-back contract, incl. fallback chunks). Prepending the header
# made content non-contiguous and broke that.
source_bytes = temp_path.read_bytes()
for i, chunk in enumerate(chunks):
assert (
"id,name,score" not in chunk.content
), f"Chunk {i} must not inline the header into content"
assert (
chunk.metadata.get("csv_header") == "id,name,score"
), f"Chunk {i} lost the CSV header from metadata"
sliced = source_bytes[chunk.byte_start : chunk.byte_end].decode("utf-8")
assert (
sliced == chunk.content
), f"Chunk {i} byte span does not slice back to content"
finally:
temp_path.unlink(missing_ok=True)

Xet Storage Details

Size:
12.6 kB
·
Xet hash:
9bd37a870595f8adc1132c70eb2db43993736b3c731b16c6b321cd72a5188481

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.