File size: 1,256 Bytes
d7228c8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | """
Unit tests for DataCleaner.
"""
import pytest
from slm.dataset.cleaner import DataCleaner
def test_data_cleaner_functions():
cleaner = DataCleaner(min_doc_chars=10, min_doc_words=2)
# 1. HTML stripping & Unicode normalization
dirty_text = "<p>Hello & World!\u00A0</p>\x00"
cleaned = cleaner.clean_document(dirty_text)
assert "<p>" not in cleaned
assert "Hello" in cleaned
assert "World!" in cleaned
# 2. Paragraph deduplication
dup_text = "Paragraph One.\n\nParagraph Two.\n\nParagraph One."
deduped = cleaner.deduplicate_paragraphs(dup_text)
assert deduped == "Paragraph One.\n\nParagraph Two."
# 3. Document validation
assert cleaner.is_valid_document("This is a valid test document.")
assert not cleaner.is_valid_document("Short")
def test_process_corpus():
cleaner = DataCleaner(min_doc_chars=10, min_doc_words=2)
corpus = [
"<h1>Title</h1>\n<p>This is a valid long document for testing.</p>",
"Too short",
"Duplicate paragraph.\n\nDuplicate paragraph."
]
retained, stats = cleaner.process_corpus(corpus)
assert stats["total_input_documents"] == 3
assert stats["retained_documents"] == 2
assert stats["filtered_documents"] == 1
|