Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Mother-AI-OS_mother /tests /test_builtin_datacraft.py
| """Tests for the built-in datacraft plugin.""" | |
| import pytest | |
| from pypdf import PdfWriter | |
| from mother.plugins.builtin.datacraft import DatacraftPlugin | |
| from mother.plugins.builtin.datacraft.parsers import ( | |
| chunk_text, | |
| detect_document_type, | |
| ) | |
| from mother.plugins.builtin.datacraft.storage import Document, DocumentStore | |
| class TestChunkText: | |
| """Tests for text chunking.""" | |
| def test_empty_text(self): | |
| """Test empty text returns empty list.""" | |
| result = chunk_text("") | |
| assert result == [] | |
| def test_short_text(self): | |
| """Test short text returns single chunk.""" | |
| result = chunk_text("Hello world") | |
| assert len(result) == 1 | |
| assert result[0] == "Hello world" | |
| def test_long_text_chunking(self): | |
| """Test long text is chunked properly.""" | |
| # Create text longer than default chunk size | |
| text = "This is a sentence. " * 100 # ~2000 chars | |
| result = chunk_text(text, chunk_size=500, overlap=100) | |
| assert len(result) > 1 | |
| # Each chunk should be reasonable size | |
| for chunk in result: | |
| assert len(chunk) <= 600 # Some tolerance for sentence boundaries | |
| def test_preserves_sentence_boundaries(self): | |
| """Test chunks break at sentence boundaries when possible.""" | |
| text = "First sentence. Second sentence. Third sentence. Fourth sentence." | |
| result = chunk_text(text, chunk_size=40, overlap=10) | |
| # Should try to break at periods | |
| for chunk in result: | |
| if not chunk.endswith("."): | |
| # Last chunk might not end with period | |
| assert chunk == result[-1] or "." in chunk | |
| class TestDetectDocumentType: | |
| """Tests for document type detection.""" | |
| def test_detect_invoice_from_filename(self): | |
| """Test invoice detection from filename.""" | |
| assert detect_document_type("invoice_2024.pdf", "") == "invoice" | |
| assert detect_document_type("Rechnung_123.pdf", "") == "invoice" | |
| def test_detect_invoice_from_content(self): | |
| """Test invoice detection from content.""" | |
| content = "Invoice Number: 12345\nTotal Amount: $500.00\nDue Date: 2024-03-15" | |
| assert detect_document_type("document.pdf", content) == "invoice" | |
| def test_detect_receipt(self): | |
| """Test receipt detection.""" | |
| assert detect_document_type("receipt_store.pdf", "") == "receipt" | |
| def test_detect_bank_statement(self): | |
| """Test bank statement detection.""" | |
| content = "Account Balance: $5,000.00\nTransaction History\nÜberweisung" | |
| assert detect_document_type("statement.pdf", content) == "bank_statement" | |
| def test_detect_contract(self): | |
| """Test contract detection.""" | |
| content = "The parties hereby agree to the following terms and conditions." | |
| assert detect_document_type("document.pdf", content) == "contract" | |
| def test_fallback_to_other(self): | |
| """Test fallback to 'other' for unknown types.""" | |
| assert detect_document_type("random.pdf", "Some random text") == "other" | |
| class TestDocumentStore: | |
| """Tests for document storage.""" | |
| def store(self, tmp_path): | |
| """Create a temporary document store.""" | |
| db_path = tmp_path / "test.db" | |
| return DocumentStore(db_path) | |
| def sample_doc(self): | |
| """Create a sample document.""" | |
| return Document( | |
| doc_id="test123", | |
| filename="test.pdf", | |
| doc_type="invoice", | |
| content="This is test content for the document.", | |
| chunks=["This is test", "content for the", "document."], | |
| metadata={"author": "Test"}, | |
| entities=[{"type": "EMAIL", "value": "test@example.com"}], | |
| file_hash="abc123", | |
| pages=1, | |
| ) | |
| def test_store_and_retrieve(self, store, sample_doc): | |
| """Test storing and retrieving a document.""" | |
| store.store_document(sample_doc) | |
| retrieved = store.get_document("test123") | |
| assert retrieved is not None | |
| assert retrieved.doc_id == "test123" | |
| assert retrieved.filename == "test.pdf" | |
| assert retrieved.doc_type == "invoice" | |
| assert len(retrieved.chunks) == 3 | |
| def test_list_documents(self, store, sample_doc): | |
| """Test listing documents.""" | |
| store.store_document(sample_doc) | |
| docs = store.list_documents() | |
| assert len(docs) == 1 | |
| assert docs[0]["doc_id"] == "test123" | |
| def test_list_documents_by_type(self, store, sample_doc): | |
| """Test filtering by document type.""" | |
| store.store_document(sample_doc) | |
| # Create another document of different type | |
| other_doc = Document( | |
| doc_id="other456", | |
| filename="other.pdf", | |
| doc_type="receipt", | |
| content="Other content", | |
| chunks=["Other content"], | |
| ) | |
| store.store_document(other_doc) | |
| invoices = store.list_documents(doc_type="invoice") | |
| assert len(invoices) == 1 | |
| assert invoices[0]["doc_type"] == "invoice" | |
| def test_delete_document(self, store, sample_doc): | |
| """Test deleting a document.""" | |
| store.store_document(sample_doc) | |
| assert store.get_document("test123") is not None | |
| deleted = store.delete_document("test123") | |
| assert deleted is True | |
| assert store.get_document("test123") is None | |
| def test_search(self, store, sample_doc): | |
| """Test full-text search.""" | |
| store.store_document(sample_doc) | |
| results = store.search("test content") | |
| assert len(results) >= 1 | |
| assert results[0].doc_id == "test123" | |
| def test_get_stats(self, store, sample_doc): | |
| """Test getting storage stats.""" | |
| store.store_document(sample_doc) | |
| stats = store.get_stats() | |
| assert stats["total_documents"] == 1 | |
| assert stats["total_chunks"] == 3 | |
| assert stats["total_entities"] == 1 | |
| def test_generate_doc_id(self, store): | |
| """Test document ID generation.""" | |
| id1 = store.generate_doc_id("content1", "file1.pdf") | |
| id2 = store.generate_doc_id("content1", "file1.pdf") | |
| id3 = store.generate_doc_id("content2", "file2.pdf") | |
| # Same content/filename should produce same ID | |
| assert id1 == id2 | |
| # Different content should produce different ID | |
| assert id1 != id3 | |
| class TestDatacraftPlugin: | |
| """Tests for the DatacraftPlugin class.""" | |
| def plugin(self, tmp_path): | |
| """Create a plugin with temporary storage.""" | |
| return DatacraftPlugin(config={"db_path": str(tmp_path / "test.db")}) | |
| def sample_pdf(self, tmp_path): | |
| """Create a sample PDF for testing.""" | |
| pdf_path = tmp_path / "sample.pdf" | |
| writer = PdfWriter() | |
| # Add a page with some content | |
| writer.add_blank_page(width=612, height=792) | |
| with open(pdf_path, "wb") as f: | |
| writer.write(f) | |
| return pdf_path | |
| def sample_txt(self, tmp_path): | |
| """Create a sample text file.""" | |
| txt_path = tmp_path / "sample.txt" | |
| txt_path.write_text( | |
| "Invoice Number: 12345\nTotal Amount: $500.00\nEmail: billing@example.com\nDue Date: 2024-03-15" | |
| ) | |
| return txt_path | |
| def test_init(self, plugin): | |
| """Test plugin initialization.""" | |
| assert plugin.manifest.plugin.name == "datacraft" | |
| assert plugin.manifest.plugin.version == "1.0.0" | |
| def test_capabilities(self, plugin): | |
| """Test plugin capabilities are defined.""" | |
| caps = plugin.get_capabilities() | |
| assert len(caps) == 8 | |
| cap_names = [c.name for c in caps] | |
| assert "process" in cap_names | |
| assert "search" in cap_names | |
| assert "tables" in cap_names | |
| assert "list" in cap_names | |
| assert "stats" in cap_names | |
| assert "delete" in cap_names | |
| def test_delete_requires_confirmation(self, plugin): | |
| """Test delete requires confirmation.""" | |
| caps = plugin.get_capabilities() | |
| delete_cap = next(c for c in caps if c.name == "delete") | |
| assert delete_cap.confirmation_required is True | |
| async def test_unknown_capability(self, plugin): | |
| """Test unknown capability returns error.""" | |
| result = await plugin.execute("unknown_cap", {}) | |
| assert result.success is False | |
| assert result.error_code == "UNKNOWN_CAPABILITY" | |
| async def test_process_txt(self, plugin, sample_txt): | |
| """Test processing a text file.""" | |
| result = await plugin.execute("process", {"path": str(sample_txt)}) | |
| assert result.success is True | |
| assert result.data["processed"] == 1 | |
| assert result.data["documents"][0]["doc_type"] == "invoice" | |
| async def test_process_no_store(self, plugin, sample_txt): | |
| """Test processing without storing.""" | |
| result = await plugin.execute( | |
| "process", | |
| {"path": str(sample_txt), "no_store": True}, | |
| ) | |
| assert result.success is True | |
| # Should not be in storage | |
| list_result = await plugin.execute("list", {}) | |
| assert list_result.data["count"] == 0 | |
| async def test_process_file_not_found(self, plugin): | |
| """Test processing non-existent file.""" | |
| result = await plugin.execute("process", {"path": "/nonexistent/file.pdf"}) | |
| assert result.success is False | |
| assert result.error_code == "FILE_NOT_FOUND" | |
| async def test_search(self, plugin, sample_txt): | |
| """Test searching documents.""" | |
| # First process a document | |
| await plugin.execute("process", {"path": str(sample_txt)}) | |
| # Then search | |
| result = await plugin.execute("search", {"query": "invoice"}) | |
| assert result.success is True | |
| assert result.data["count"] >= 1 | |
| async def test_list(self, plugin, sample_txt): | |
| """Test listing documents.""" | |
| await plugin.execute("process", {"path": str(sample_txt)}) | |
| result = await plugin.execute("list", {}) | |
| assert result.success is True | |
| assert result.data["count"] == 1 | |
| async def test_stats(self, plugin, sample_txt): | |
| """Test getting stats.""" | |
| await plugin.execute("process", {"path": str(sample_txt)}) | |
| result = await plugin.execute("stats", {}) | |
| assert result.success is True | |
| assert result.data["total_documents"] == 1 | |
| async def test_delete(self, plugin, sample_txt): | |
| """Test deleting a document.""" | |
| # Process | |
| process_result = await plugin.execute("process", {"path": str(sample_txt)}) | |
| doc_id = process_result.data["documents"][0]["doc_id"] | |
| # Delete | |
| result = await plugin.execute("delete", {"doc_id": doc_id}) | |
| assert result.success is True | |
| # Verify deleted | |
| list_result = await plugin.execute("list", {}) | |
| assert list_result.data["count"] == 0 | |
| async def test_delete_not_found(self, plugin): | |
| """Test deleting non-existent document.""" | |
| result = await plugin.execute("delete", {"doc_id": "nonexistent"}) | |
| assert result.success is False | |
| assert result.error_code == "NOT_FOUND" | |
| async def test_get(self, plugin, sample_txt): | |
| """Test getting a document by ID.""" | |
| # Process | |
| process_result = await plugin.execute("process", {"path": str(sample_txt)}) | |
| doc_id = process_result.data["documents"][0]["doc_id"] | |
| # Get | |
| result = await plugin.execute("get", {"doc_id": doc_id}) | |
| assert result.success is True | |
| assert result.data["doc_id"] == doc_id | |
| async def test_get_not_found(self, plugin): | |
| """Test getting non-existent document.""" | |
| result = await plugin.execute("get", {"doc_id": "nonexistent"}) | |
| assert result.success is False | |
| assert result.error_code == "NOT_FOUND" | |
| async def test_tables(self, plugin, sample_txt): | |
| """Test extracting tables.""" | |
| result = await plugin.execute("tables", {"path": str(sample_txt)}) | |
| assert result.success is True | |
| # Text file likely has no tables | |
| assert "tables" in result.data | |
| async def test_tables_file_not_found(self, plugin): | |
| """Test tables with missing file.""" | |
| result = await plugin.execute("tables", {"path": "/nonexistent.pdf"}) | |
| assert result.success is False | |
| assert result.error_code == "FILE_NOT_FOUND" | |
| async def test_graph(self, plugin, sample_txt): | |
| """Test getting knowledge graph.""" | |
| # Process | |
| process_result = await plugin.execute("process", {"path": str(sample_txt)}) | |
| doc_id = process_result.data["documents"][0]["doc_id"] | |
| # Get graph | |
| result = await plugin.execute("graph", {"doc_id": doc_id}) | |
| assert result.success is True | |
| assert "entities" in result.data | |
| assert "relationships" in result.data | |
| async def test_process_directory(self, plugin, tmp_path): | |
| """Test processing a directory.""" | |
| # Create multiple files | |
| (tmp_path / "doc1.txt").write_text("First document content") | |
| (tmp_path / "doc2.txt").write_text("Second document content") | |
| result = await plugin.execute("process", {"path": str(tmp_path)}) | |
| assert result.success is True | |
| assert result.data["processed"] == 2 | |
| async def test_unsupported_file_type(self, plugin, tmp_path): | |
| """Test handling unsupported file type.""" | |
| unsupported = tmp_path / "file.xyz" | |
| unsupported.write_text("content") | |
| result = await plugin.execute("process", {"path": str(unsupported)}) | |
| # Should fail gracefully | |
| assert result.data["failed"] == 1 | |
Xet Storage Details
- Size:
- 14.2 kB
- Xet hash:
- 11103c230dc0db779a3fc002c71e135a8e0269053aea1313ec3852684fd1b4c6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.