Spaces:
Running on Zero
Running on Zero
Download tests/test_ingest.py from Varun2007/document-qa-system: direct link, hf CLI and curl.
- Browser
- Download file 2.72 kB
-
https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/tests/test_ingest.py
- Command line
-
hf download hf://spaces/Varun2007/document-qa-system/tests/test_ingest.py
-
curl -L -o test_ingest.py https://huggingface.co/spaces/Varun2007/document-qa-system/resolve/main/tests/test_ingest.py
2.72 kB
| """Unit tests for Document Ingestion and Chunking.""" | |
| import os | |
| import unittest | |
| from ingest import ( | |
| DocumentIngestionPipeline, | |
| DocumentChunk, | |
| count_tokens, | |
| table_to_markdown, | |
| ) | |
| class TestIngestion(unittest.TestCase): | |
| def setUp(self): | |
| self.pipeline = DocumentIngestionPipeline(chunk_size=300, chunk_overlap=40) | |
| self.test_txt_path = "tests/test_doc.txt" | |
| with open(self.test_txt_path, "w", encoding="utf-8") as f: | |
| f.write( | |
| "# Executive Summary\n" | |
| "Company revenue expanded by 20% in fiscal year 2023.\n\n" | |
| "## Financial Details\n" | |
| "Operating expenses were managed efficiently at $50 million.\n" | |
| "Capital expenditure was focused on new engineering hubs.\n" | |
| ) | |
| def tearDown(self): | |
| if os.path.exists(self.test_txt_path): | |
| os.remove(self.test_txt_path) | |
| def test_token_counter(self): | |
| text = "Hello world this is a test document." | |
| count = count_tokens(text) | |
| self.assertGreater(count, 0) | |
| self.assertLess(count, 30) | |
| def test_table_to_markdown(self): | |
| rows = [ | |
| ["Metric", "Value"], | |
| ["Revenue", "$100M"], | |
| ["Margin", "25%"] | |
| ] | |
| md = table_to_markdown(rows) | |
| self.assertIn("| Metric | Value |", md) | |
| self.assertIn("| Revenue | $100M |", md) | |
| self.assertIn("| Margin | 25% |", md) | |
| def test_txt_ingestion_and_chunking(self): | |
| doc = self.pipeline.ingest_file(self.test_txt_path) | |
| self.assertEqual(doc.file_type, "txt") | |
| self.assertGreater(len(doc.chunks), 0) | |
| # Verify metadata | |
| first_chunk = doc.chunks[0] | |
| self.assertIsInstance(first_chunk, DocumentChunk) | |
| self.assertEqual(first_chunk.page_number, 1) | |
| self.assertIn("Summary", first_chunk.section_title) | |
| self.assertGreater(first_chunk.token_count, 0) | |
| def test_sample_files_exist_and_parse(self): | |
| samples = [ | |
| "data/sample_financial_report.pdf", | |
| "data/sample_cloud_architecture.docx", | |
| "data/sample_security_policy.txt" | |
| ] | |
| for s in samples: | |
| if os.path.exists(s): | |
| doc = self.pipeline.ingest_file(s) | |
| self.assertGreater(len(doc.chunks), 0, f"Expected chunks for {s}") | |
| for chunk in doc.chunks: | |
| self.assertIsNotNone(chunk.text) | |
| self.assertGreater(len(chunk.text.strip()), 0) | |
| self.assertGreaterEqual(chunk.page_number, 1) | |
| self.assertTrue(len(chunk.section_title) > 0) | |
| if __name__ == "__main__": | |
| unittest.main() | |