Spaces:
Sleeping
Sleeping
File size: 1,859 Bytes
da21448 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | from pathlib import Path
import pytest
from techdocs_graphrag.ingestion import (
ParagraphSplitter,
SourceDocument,
chunk_documents,
load_documents,
)
def test_splitter_respects_chunk_size_and_covers_every_paragraph() -> None:
paragraphs = [f"P{index:02d} " + "mot " * 40 for index in range(10)]
chunks = ParagraphSplitter(chunk_size=120, chunk_overlap=20).split("\n\n".join(paragraphs))
joined = " ".join(chunks)
assert all(len(chunk) <= 120 for chunk in chunks)
assert all(f"P{index:02d}" in joined for index in range(10))
def test_splitter_keeps_short_paragraphs_together() -> None:
chunks = ParagraphSplitter(chunk_size=100, chunk_overlap=10).split("Un.\n\nDeux.\n\nTrois.")
assert chunks == ["Un.\n\nDeux.\n\nTrois."]
def test_splitter_rejects_overlap_larger_than_chunk() -> None:
with pytest.raises(ValueError):
ParagraphSplitter(chunk_size=10, chunk_overlap=10)
def test_load_documents_reads_supported_files_recursively(tmp_path: Path) -> None:
(tmp_path / "guides").mkdir()
(tmp_path / "intro.md").write_text("Intro", encoding="utf-8")
(tmp_path / "guides" / "setup.md").write_text("Setup", encoding="utf-8")
(tmp_path / "image.png").write_bytes(b"\x89PNG")
documents = load_documents(tmp_path)
assert [document.name for document in documents] == ["guides/setup.md", "intro.md"]
def test_chunk_ids_are_deterministic_and_unique() -> None:
documents = [SourceDocument("a.md", "Alpha.\n\nBeta."), SourceDocument("b.md", "Alpha.")]
splitter = ParagraphSplitter(chunk_size=8, chunk_overlap=0)
first_run = chunk_documents(documents, splitter)
second_run = chunk_documents(documents, splitter)
assert [chunk.id for chunk in first_run] == [chunk.id for chunk in second_run]
assert len({chunk.id for chunk in first_run}) == len(first_run)
|