Spaces:
Sleeping
Sleeping
File size: 3,384 Bytes
331bcca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | import sys
from unittest.mock import MagicMock
import unittest
# Mock fitz module before importing pdf_processor since it's not available in the test environment
sys.modules["fitz"] = MagicMock()
from services.pdf_processor import PDFProcessor
class TestPDFProcessorCleanText(unittest.TestCase):
def setUp(self):
self.processor = PDFProcessor()
def test_clean_text_none(self):
"""Test that None input returns empty string."""
self.assertEqual(self.processor.clean_text(None), "")
def test_clean_text_empty(self):
"""Test that empty string input returns empty string."""
self.assertEqual(self.processor.clean_text(""), "")
def test_clean_text_artifacts(self):
"""Test removal of various artifacts."""
# Footer artifact
self.assertEqual(self.processor.clean_text("NotebookLM"), "")
# Note: sub("NotebookLM", "") leaves two spaces if it was surrounded by spaces
# "Text NotebookLM Text" -> "Text Text" -> normalized to "Text Text" by MULTIPLE_SPACES
self.assertEqual(self.processor.clean_text("Text NotebookLM Text"), "Text Text")
# Case-insensitive check
self.assertEqual(self.processor.clean_text("notebooklm"), "")
self.assertEqual(self.processor.clean_text("NOTEBOOKLM"), "")
self.assertEqual(self.processor.clean_text("NoTeBoOkLm"), "")
# Citation
self.assertEqual(self.processor.clean_text("Fact [1]"), "Fact")
self.assertEqual(self.processor.clean_text("Fact [12]"), "Fact")
# Page marker
self.assertEqual(self.processor.clean_text("--- PAGE 3 ---"), "")
# "Text --- PAGE 3 --- Text" -> "Text Text" -> "Text Text"
self.assertEqual(self.processor.clean_text("Text --- PAGE 3 --- Text"), "Text Text")
def test_clean_text_page_numbers(self):
"""Test that lines with only numbers are removed."""
# Line with only number
self.assertEqual(self.processor.clean_text("123"), "")
self.assertEqual(self.processor.clean_text("Text\n123\nText"), "Text\nText")
# Line with number and spaces
self.assertEqual(self.processor.clean_text(" 123 "), "")
def test_clean_text_whitespace(self):
"""Test whitespace normalization."""
# Multiple spaces
self.assertEqual(self.processor.clean_text("A B"), "A B")
# Multiple newlines (Note: implementation removes empty lines first)
self.assertEqual(self.processor.clean_text("A\n\n\nB"), "A\nB")
# Leading/Trailing whitespace
self.assertEqual(self.processor.clean_text(" A "), "A")
def test_clean_text_combined(self):
"""Test a combination of cleaning operations."""
text = "NotebookLM\n[1] Citation.\n--- PAGE 1 ---\n1\nReal Text."
# 1. "NotebookLM" -> "" -> Empty line -> Removed.
# 2. "[1] Citation." -> " Citation." -> Kept.
# 3. "--- PAGE 1 ---" -> "" -> Empty line -> Removed.
# 4. "1" -> Digit -> Removed.
# 5. "Real Text." -> Kept.
# Result joined: " Citation.\nReal Text."
# Whitespace norm: " Citation.\nReal Text." (no multiple spaces)
# Final strip: "Citation.\nReal Text."
expected = "Citation.\nReal Text."
self.assertEqual(self.processor.clean_text(text), expected)
if __name__ == "__main__":
unittest.main()
|