Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /AmberLee2427_nancy-brain /tests /test_chunking.py
| from pathlib import Path | |
| from chunky import ChunkPipeline, ChunkerConfig, Document | |
| from nancy_brain.chunking import strip_chunk_suffix | |
| def _make_content(lines: int) -> str: | |
| return "\n".join(f"line {i}" for i in range(1, lines + 1)) | |
| def test_pipeline_respects_line_window(tmp_path: Path) -> None: | |
| pipeline = ChunkPipeline() | |
| lines_per_chunk = 40 | |
| line_overlap = 5 | |
| config = ChunkerConfig(lines_per_chunk=lines_per_chunk, line_overlap=line_overlap) | |
| path = tmp_path / "sample.py" | |
| path.write_text(_make_content(120), encoding="utf-8") | |
| doc_id = "cat/repo/sample.py" | |
| document = Document(path=path, content=path.read_text(), metadata={"doc_id": doc_id}) | |
| chunks = pipeline.chunk_documents([document], config=config) | |
| step = max(1, lines_per_chunk - line_overlap) | |
| total_lines = 120 | |
| expected = 1 if total_lines <= lines_per_chunk else ((total_lines - lines_per_chunk + step - 1) // step) + 1 | |
| assert len(chunks) == expected | |
| assert chunks[0].chunk_id == f"{doc_id}#chunk-0000" | |
| assert chunks[0].metadata["line_start"] == 1 | |
| assert chunks[0].metadata["line_end"] == lines_per_chunk | |
| for previous, current in zip(chunks, chunks[1:]): | |
| assert current.metadata["line_start"] == previous.metadata["line_end"] - line_overlap + 1 | |
| assert previous.metadata["chunk_count"] == expected | |
| assert current.metadata["chunk_count"] == expected | |
| assert previous.metadata["source_document"] == doc_id | |
| assert current.metadata["source_document"] == doc_id | |
| assert chunks[-1].metadata["line_end"] == total_lines | |
| def test_pipeline_handles_empty_files(tmp_path: Path) -> None: | |
| pipeline = ChunkPipeline() | |
| path = tmp_path / "empty.txt" | |
| path.write_text("", encoding="utf-8") | |
| document = Document(path=path, content="", metadata={"doc_id": "docs/empty.txt"}) | |
| chunks = pipeline.chunk_documents([document]) | |
| assert len(chunks) == 1 | |
| chunk = chunks[0] | |
| assert chunk.chunk_id == "docs/empty.txt#chunk-0000" | |
| assert chunk.text == "" | |
| assert chunk.metadata["line_start"] == 0 | |
| assert chunk.metadata["line_end"] == 0 | |
| assert chunk.metadata["chunk_count"] == 1 | |
| def test_strip_chunk_suffix(): | |
| cases = [ | |
| ("pkg/module.py#chunk-0002", "pkg/module.py"), | |
| ("pkg/module.py::chunk-0002", "pkg/module.py"), | |
| ("pkg/module.py|chunk:3", "pkg/module.py"), | |
| ("pkg/module.py", "pkg/module.py"), | |
| ] | |
| for doc_id, expected in cases: | |
| assert strip_chunk_suffix(doc_id) == expected | |
Xet Storage Details
- Size:
- 2.52 kB
- Xet hash:
- cd3df05e8d0030d18a07ec020e94cc2ec5c36cb8a56fe1dd6430baea6ce56506
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.