tests/test_parser.py

from backend.parser import chunk_text, chunk_pages, page_offsets, char_pos_to_page


def test_chunk_text_covers_all_words():
    text = " ".join(f"w{i}" for i in range(1200))
    chunks = chunk_text(text, size=512, overlap=50)
    assert len(chunks) == 3
    assert chunks[0].split()[0] == "w0"
    assert chunks[-1].split()[-1] == "w1199"


def test_chunk_pages_carry_page_numbers():
    pages = ["first page text here", "", "third page text here"]
    chunks = chunk_pages(pages)
    assert [c["page"] for c in chunks] == [1, 3]  # empty page skipped
    assert chunks[0]["text"] == "first page text here"


def test_page_offsets_and_mapping():
    pages = ["aaaa", "bbbbbb", "cc"]
    offsets = page_offsets(pages)
    assert offsets == [0, 5, 12]  # +1 for each joining newline
    joined = "\n".join(pages)
    assert char_pos_to_page(0, offsets) == 1
    assert char_pos_to_page(5, offsets) == 2
    assert char_pos_to_page(len(joined) - 1, offsets) == 3


def test_char_pos_to_page_handles_none():
    assert char_pos_to_page(None, [0, 5]) is None
    assert char_pos_to_page(3, []) is None