Spaces:
Running
Running
| import sys | |
| from unittest.mock import MagicMock | |
| import unittest | |
| # Mock fitz module before importing pdf_processor since it's not available in the test environment | |
| sys.modules["fitz"] = MagicMock() | |
| from services.pdf_processor import PDFProcessor | |
| class TestPDFProcessorCleanText(unittest.TestCase): | |
| def setUp(self): | |
| self.processor = PDFProcessor() | |
| def test_clean_text_none(self): | |
| """Test that None input returns empty string.""" | |
| self.assertEqual(self.processor.clean_text(None), "") | |
| def test_clean_text_empty(self): | |
| """Test that empty string input returns empty string.""" | |
| self.assertEqual(self.processor.clean_text(""), "") | |
| def test_clean_text_artifacts(self): | |
| """Test removal of various artifacts.""" | |
| # Footer artifact | |
| self.assertEqual(self.processor.clean_text("NotebookLM"), "") | |
| # Note: sub("NotebookLM", "") leaves two spaces if it was surrounded by spaces | |
| # "Text NotebookLM Text" -> "Text Text" -> normalized to "Text Text" by MULTIPLE_SPACES | |
| self.assertEqual(self.processor.clean_text("Text NotebookLM Text"), "Text Text") | |
| # Case-insensitive check | |
| self.assertEqual(self.processor.clean_text("notebooklm"), "") | |
| self.assertEqual(self.processor.clean_text("NOTEBOOKLM"), "") | |
| self.assertEqual(self.processor.clean_text("NoTeBoOkLm"), "") | |
| # Citation | |
| self.assertEqual(self.processor.clean_text("Fact [1]"), "Fact") | |
| self.assertEqual(self.processor.clean_text("Fact [12]"), "Fact") | |
| # Page marker | |
| self.assertEqual(self.processor.clean_text("--- PAGE 3 ---"), "") | |
| # "Text --- PAGE 3 --- Text" -> "Text Text" -> "Text Text" | |
| self.assertEqual(self.processor.clean_text("Text --- PAGE 3 --- Text"), "Text Text") | |
| def test_clean_text_page_numbers(self): | |
| """Test that lines with only numbers are removed.""" | |
| # Line with only number | |
| self.assertEqual(self.processor.clean_text("123"), "") | |
| self.assertEqual(self.processor.clean_text("Text\n123\nText"), "Text\nText") | |
| # Line with number and spaces | |
| self.assertEqual(self.processor.clean_text(" 123 "), "") | |
| def test_clean_text_whitespace(self): | |
| """Test whitespace normalization.""" | |
| # Multiple spaces | |
| self.assertEqual(self.processor.clean_text("A B"), "A B") | |
| # Multiple newlines (Note: implementation removes empty lines first) | |
| self.assertEqual(self.processor.clean_text("A\n\n\nB"), "A\nB") | |
| # Leading/Trailing whitespace | |
| self.assertEqual(self.processor.clean_text(" A "), "A") | |
| def test_clean_text_combined(self): | |
| """Test a combination of cleaning operations.""" | |
| text = "NotebookLM\n[1] Citation.\n--- PAGE 1 ---\n1\nReal Text." | |
| # 1. "NotebookLM" -> "" -> Empty line -> Removed. | |
| # 2. "[1] Citation." -> " Citation." -> Kept. | |
| # 3. "--- PAGE 1 ---" -> "" -> Empty line -> Removed. | |
| # 4. "1" -> Digit -> Removed. | |
| # 5. "Real Text." -> Kept. | |
| # Result joined: " Citation.\nReal Text." | |
| # Whitespace norm: " Citation.\nReal Text." (no multiple spaces) | |
| # Final strip: "Citation.\nReal Text." | |
| expected = "Citation.\nReal Text." | |
| self.assertEqual(self.processor.clean_text(text), expected) | |
| if __name__ == "__main__": | |
| unittest.main() | |