document-qa-system / tests /test_ingest.py
Varun2007's picture
Production Document Question Answering (DQA) ready for Hugging Face Spaces
d3d09d6
Raw History Blame Contribute Delete
2.72 kB
"""Unit tests for Document Ingestion and Chunking."""
import os
import unittest
from ingest import (
DocumentIngestionPipeline,
DocumentChunk,
count_tokens,
table_to_markdown,
)
class TestIngestion(unittest.TestCase):
def setUp(self):
self.pipeline = DocumentIngestionPipeline(chunk_size=300, chunk_overlap=40)
self.test_txt_path = "tests/test_doc.txt"
with open(self.test_txt_path, "w", encoding="utf-8") as f:
f.write(
"# Executive Summary\n"
"Company revenue expanded by 20% in fiscal year 2023.\n\n"
"## Financial Details\n"
"Operating expenses were managed efficiently at $50 million.\n"
"Capital expenditure was focused on new engineering hubs.\n"
)
def tearDown(self):
if os.path.exists(self.test_txt_path):
os.remove(self.test_txt_path)
def test_token_counter(self):
text = "Hello world this is a test document."
count = count_tokens(text)
self.assertGreater(count, 0)
self.assertLess(count, 30)
def test_table_to_markdown(self):
rows = [
["Metric", "Value"],
["Revenue", "$100M"],
["Margin", "25%"]
]
md = table_to_markdown(rows)
self.assertIn("| Metric | Value |", md)
self.assertIn("| Revenue | $100M |", md)
self.assertIn("| Margin | 25% |", md)
def test_txt_ingestion_and_chunking(self):
doc = self.pipeline.ingest_file(self.test_txt_path)
self.assertEqual(doc.file_type, "txt")
self.assertGreater(len(doc.chunks), 0)
# Verify metadata
first_chunk = doc.chunks[0]
self.assertIsInstance(first_chunk, DocumentChunk)
self.assertEqual(first_chunk.page_number, 1)
self.assertIn("Summary", first_chunk.section_title)
self.assertGreater(first_chunk.token_count, 0)
def test_sample_files_exist_and_parse(self):
samples = [
"data/sample_financial_report.pdf",
"data/sample_cloud_architecture.docx",
"data/sample_security_policy.txt"
]
for s in samples:
if os.path.exists(s):
doc = self.pipeline.ingest_file(s)
self.assertGreater(len(doc.chunks), 0, f"Expected chunks for {s}")
for chunk in doc.chunks:
self.assertIsNotNone(chunk.text)
self.assertGreater(len(chunk.text.strip()), 0)
self.assertGreaterEqual(chunk.page_number, 1)
self.assertTrue(len(chunk.section_title) > 0)
if __name__ == "__main__":
unittest.main()