"""Unit tests for Document Ingestion and Chunking.""" import os import unittest from ingest import ( DocumentIngestionPipeline, DocumentChunk, count_tokens, table_to_markdown, ) class TestIngestion(unittest.TestCase): def setUp(self): self.pipeline = DocumentIngestionPipeline(chunk_size=300, chunk_overlap=40) self.test_txt_path = "tests/test_doc.txt" with open(self.test_txt_path, "w", encoding="utf-8") as f: f.write( "# Executive Summary\n" "Company revenue expanded by 20% in fiscal year 2023.\n\n" "## Financial Details\n" "Operating expenses were managed efficiently at $50 million.\n" "Capital expenditure was focused on new engineering hubs.\n" ) def tearDown(self): if os.path.exists(self.test_txt_path): os.remove(self.test_txt_path) def test_token_counter(self): text = "Hello world this is a test document." count = count_tokens(text) self.assertGreater(count, 0) self.assertLess(count, 30) def test_table_to_markdown(self): rows = [ ["Metric", "Value"], ["Revenue", "$100M"], ["Margin", "25%"] ] md = table_to_markdown(rows) self.assertIn("| Metric | Value |", md) self.assertIn("| Revenue | $100M |", md) self.assertIn("| Margin | 25% |", md) def test_txt_ingestion_and_chunking(self): doc = self.pipeline.ingest_file(self.test_txt_path) self.assertEqual(doc.file_type, "txt") self.assertGreater(len(doc.chunks), 0) # Verify metadata first_chunk = doc.chunks[0] self.assertIsInstance(first_chunk, DocumentChunk) self.assertEqual(first_chunk.page_number, 1) self.assertIn("Summary", first_chunk.section_title) self.assertGreater(first_chunk.token_count, 0) def test_sample_files_exist_and_parse(self): samples = [ "data/sample_financial_report.pdf", "data/sample_cloud_architecture.docx", "data/sample_security_policy.txt" ] for s in samples: if os.path.exists(s): doc = self.pipeline.ingest_file(s) self.assertGreater(len(doc.chunks), 0, f"Expected chunks for {s}") for chunk in doc.chunks: self.assertIsNotNone(chunk.text) self.assertGreater(len(chunk.text.strip()), 0) self.assertGreaterEqual(chunk.page_number, 1) self.assertTrue(len(chunk.section_title) > 0) if __name__ == "__main__": unittest.main()