Spaces:
Sleeping
Sleeping
| from pathlib import Path | |
| import pytest | |
| from techdocs_graphrag.ingestion import ( | |
| ParagraphSplitter, | |
| SourceDocument, | |
| chunk_documents, | |
| load_documents, | |
| ) | |
| def test_splitter_respects_chunk_size_and_covers_every_paragraph() -> None: | |
| paragraphs = [f"P{index:02d} " + "mot " * 40 for index in range(10)] | |
| chunks = ParagraphSplitter(chunk_size=120, chunk_overlap=20).split("\n\n".join(paragraphs)) | |
| joined = " ".join(chunks) | |
| assert all(len(chunk) <= 120 for chunk in chunks) | |
| assert all(f"P{index:02d}" in joined for index in range(10)) | |
| def test_splitter_keeps_short_paragraphs_together() -> None: | |
| chunks = ParagraphSplitter(chunk_size=100, chunk_overlap=10).split("Un.\n\nDeux.\n\nTrois.") | |
| assert chunks == ["Un.\n\nDeux.\n\nTrois."] | |
| def test_splitter_rejects_overlap_larger_than_chunk() -> None: | |
| with pytest.raises(ValueError): | |
| ParagraphSplitter(chunk_size=10, chunk_overlap=10) | |
| def test_load_documents_reads_supported_files_recursively(tmp_path: Path) -> None: | |
| (tmp_path / "guides").mkdir() | |
| (tmp_path / "intro.md").write_text("Intro", encoding="utf-8") | |
| (tmp_path / "guides" / "setup.md").write_text("Setup", encoding="utf-8") | |
| (tmp_path / "image.png").write_bytes(b"\x89PNG") | |
| documents = load_documents(tmp_path) | |
| assert [document.name for document in documents] == ["guides/setup.md", "intro.md"] | |
| def test_chunk_ids_are_deterministic_and_unique() -> None: | |
| documents = [SourceDocument("a.md", "Alpha.\n\nBeta."), SourceDocument("b.md", "Alpha.")] | |
| splitter = ParagraphSplitter(chunk_size=8, chunk_overlap=0) | |
| first_run = chunk_documents(documents, splitter) | |
| second_run = chunk_documents(documents, splitter) | |
| assert [chunk.id for chunk in first_run] == [chunk.id for chunk in second_run] | |
| assert len({chunk.id for chunk in first_run}) == len(first_run) | |