from pathlib import Path import pytest from techdocs_graphrag.ingestion import ( ParagraphSplitter, SourceDocument, chunk_documents, load_documents, ) def test_splitter_respects_chunk_size_and_covers_every_paragraph() -> None: paragraphs = [f"P{index:02d} " + "mot " * 40 for index in range(10)] chunks = ParagraphSplitter(chunk_size=120, chunk_overlap=20).split("\n\n".join(paragraphs)) joined = " ".join(chunks) assert all(len(chunk) <= 120 for chunk in chunks) assert all(f"P{index:02d}" in joined for index in range(10)) def test_splitter_keeps_short_paragraphs_together() -> None: chunks = ParagraphSplitter(chunk_size=100, chunk_overlap=10).split("Un.\n\nDeux.\n\nTrois.") assert chunks == ["Un.\n\nDeux.\n\nTrois."] def test_splitter_rejects_overlap_larger_than_chunk() -> None: with pytest.raises(ValueError): ParagraphSplitter(chunk_size=10, chunk_overlap=10) def test_load_documents_reads_supported_files_recursively(tmp_path: Path) -> None: (tmp_path / "guides").mkdir() (tmp_path / "intro.md").write_text("Intro", encoding="utf-8") (tmp_path / "guides" / "setup.md").write_text("Setup", encoding="utf-8") (tmp_path / "image.png").write_bytes(b"\x89PNG") documents = load_documents(tmp_path) assert [document.name for document in documents] == ["guides/setup.md", "intro.md"] def test_chunk_ids_are_deterministic_and_unique() -> None: documents = [SourceDocument("a.md", "Alpha.\n\nBeta."), SourceDocument("b.md", "Alpha.")] splitter = ParagraphSplitter(chunk_size=8, chunk_overlap=0) first_run = chunk_documents(documents, splitter) second_run = chunk_documents(documents, splitter) assert [chunk.id for chunk in first_run] == [chunk.id for chunk in second_run] assert len({chunk.id for chunk in first_run}) == len(first_run)