File size: 2,720 Bytes
d3d09d6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
"""Unit tests for Document Ingestion and Chunking."""

import os
import unittest
from ingest import (
    DocumentIngestionPipeline,
    DocumentChunk,
    count_tokens,
    table_to_markdown,
)


class TestIngestion(unittest.TestCase):
    def setUp(self):
        self.pipeline = DocumentIngestionPipeline(chunk_size=300, chunk_overlap=40)
        self.test_txt_path = "tests/test_doc.txt"
        with open(self.test_txt_path, "w", encoding="utf-8") as f:
            f.write(
                "# Executive Summary\n"
                "Company revenue expanded by 20% in fiscal year 2023.\n\n"
                "## Financial Details\n"
                "Operating expenses were managed efficiently at $50 million.\n"
                "Capital expenditure was focused on new engineering hubs.\n"
            )

    def tearDown(self):
        if os.path.exists(self.test_txt_path):
            os.remove(self.test_txt_path)

    def test_token_counter(self):
        text = "Hello world this is a test document."
        count = count_tokens(text)
        self.assertGreater(count, 0)
        self.assertLess(count, 30)

    def test_table_to_markdown(self):
        rows = [
            ["Metric", "Value"],
            ["Revenue", "$100M"],
            ["Margin", "25%"]
        ]
        md = table_to_markdown(rows)
        self.assertIn("| Metric | Value |", md)
        self.assertIn("| Revenue | $100M |", md)
        self.assertIn("| Margin | 25% |", md)

    def test_txt_ingestion_and_chunking(self):
        doc = self.pipeline.ingest_file(self.test_txt_path)
        self.assertEqual(doc.file_type, "txt")
        self.assertGreater(len(doc.chunks), 0)
        
        # Verify metadata
        first_chunk = doc.chunks[0]
        self.assertIsInstance(first_chunk, DocumentChunk)
        self.assertEqual(first_chunk.page_number, 1)
        self.assertIn("Summary", first_chunk.section_title)
        self.assertGreater(first_chunk.token_count, 0)

    def test_sample_files_exist_and_parse(self):
        samples = [
            "data/sample_financial_report.pdf",
            "data/sample_cloud_architecture.docx",
            "data/sample_security_policy.txt"
        ]
        for s in samples:
            if os.path.exists(s):
                doc = self.pipeline.ingest_file(s)
                self.assertGreater(len(doc.chunks), 0, f"Expected chunks for {s}")
                for chunk in doc.chunks:
                    self.assertIsNotNone(chunk.text)
                    self.assertGreater(len(chunk.text.strip()), 0)
                    self.assertGreaterEqual(chunk.page_number, 1)
                    self.assertTrue(len(chunk.section_title) > 0)


if __name__ == "__main__":
    unittest.main()