Spaces:
Running on Zero
Running on Zero
File size: 2,720 Bytes
d3d09d6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | """Unit tests for Document Ingestion and Chunking."""
import os
import unittest
from ingest import (
DocumentIngestionPipeline,
DocumentChunk,
count_tokens,
table_to_markdown,
)
class TestIngestion(unittest.TestCase):
def setUp(self):
self.pipeline = DocumentIngestionPipeline(chunk_size=300, chunk_overlap=40)
self.test_txt_path = "tests/test_doc.txt"
with open(self.test_txt_path, "w", encoding="utf-8") as f:
f.write(
"# Executive Summary\n"
"Company revenue expanded by 20% in fiscal year 2023.\n\n"
"## Financial Details\n"
"Operating expenses were managed efficiently at $50 million.\n"
"Capital expenditure was focused on new engineering hubs.\n"
)
def tearDown(self):
if os.path.exists(self.test_txt_path):
os.remove(self.test_txt_path)
def test_token_counter(self):
text = "Hello world this is a test document."
count = count_tokens(text)
self.assertGreater(count, 0)
self.assertLess(count, 30)
def test_table_to_markdown(self):
rows = [
["Metric", "Value"],
["Revenue", "$100M"],
["Margin", "25%"]
]
md = table_to_markdown(rows)
self.assertIn("| Metric | Value |", md)
self.assertIn("| Revenue | $100M |", md)
self.assertIn("| Margin | 25% |", md)
def test_txt_ingestion_and_chunking(self):
doc = self.pipeline.ingest_file(self.test_txt_path)
self.assertEqual(doc.file_type, "txt")
self.assertGreater(len(doc.chunks), 0)
# Verify metadata
first_chunk = doc.chunks[0]
self.assertIsInstance(first_chunk, DocumentChunk)
self.assertEqual(first_chunk.page_number, 1)
self.assertIn("Summary", first_chunk.section_title)
self.assertGreater(first_chunk.token_count, 0)
def test_sample_files_exist_and_parse(self):
samples = [
"data/sample_financial_report.pdf",
"data/sample_cloud_architecture.docx",
"data/sample_security_policy.txt"
]
for s in samples:
if os.path.exists(s):
doc = self.pipeline.ingest_file(s)
self.assertGreater(len(doc.chunks), 0, f"Expected chunks for {s}")
for chunk in doc.chunks:
self.assertIsNotNone(chunk.text)
self.assertGreater(len(chunk.text.strip()), 0)
self.assertGreaterEqual(chunk.page_number, 1)
self.assertTrue(len(chunk.section_title) > 0)
if __name__ == "__main__":
unittest.main()
|