Spaces:
Sleeping
Sleeping
Download KnowledgeService/tests/test_knowledge.py from dellweb3/workos-office: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/tests/test_knowledge.py
- Command line
-
hf download hf://spaces/dellweb3/workos-office/KnowledgeService/tests/test_knowledge.py
-
curl -L -o test_knowledge.py https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/tests/test_knowledge.py
12.6 kB
| """Run from base1: python -m unittest discover -s KnowledgeService/tests -v.""" | |
| import io | |
| import json | |
| import os | |
| import sys | |
| import tempfile | |
| import unittest | |
| import zipfile | |
| from pathlib import Path | |
| from sqlalchemy import delete, func, select | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT / "APIGateway")) | |
| from WorkspaceService.workspace_service import WorkspaceService | |
| from KnowledgeService import KnowledgeService, KnowledgeError | |
| from KnowledgeService.models import Chunk, Document, Grant | |
| from KnowledgeService.parsing import ParseError, extract | |
| A = {"tenant_id": "demo-tenant", "user_id": "u-lin"} | |
| B = {"tenant_id": "demo-tenant", "user_id": "u-zhou"} | |
| class KnowledgeTests(unittest.TestCase): | |
| def setUp(self): | |
| self.tmp = tempfile.TemporaryDirectory() | |
| self.ws = WorkspaceService(database_url="sqlite:///" + self.tmp.name + "/test.db", | |
| file_root=self.tmp.name + "/files", auto_create_schema=True) | |
| self.kb = KnowledgeService(self.ws) | |
| self.wa = self.ws.create(A, "归档事项", "matter-a")[0]["workspace_id"] | |
| self.wb = self.ws.create(B, "设备盘点", "matter-b")[0]["workspace_id"] | |
| def tearDown(self): | |
| self.ws.engine.dispose() | |
| self.tmp.cleanup() | |
| def ingest(self, user=A, wid=None, name="notice.md", content="资料归档截止日期 PRIVATE_A 2026年10月20日"): | |
| wid = wid or self.wa | |
| a, _ = self.ws.attachment(user, wid, name, content.encode() if isinstance(content, str) else content, name) | |
| sent = self.ws.dispatch_pending(self.kb.consume_event, event_types={"document.ingest.requested"}) | |
| self.assertEqual(sent, 1) | |
| return a | |
| def search(self, query, user=A, wid=None, bases=None): | |
| return self.kb.search(user, wid or self.wa, query, bases or [], 3)["items"] | |
| def test_five_shared_foundations_and_private_base(self): | |
| bases = self.kb.bases(A)["items"] | |
| self.assertEqual(len([b for b in bases if b["visibility"] == "shared"]), 5) | |
| self.assertEqual(len([b for b in bases if b["visibility"] == "personal"]), 1) | |
| from KnowledgeService.seed import inventory | |
| self.assertEqual(len(self.kb.documents(A, self.wa)["items"]), inventory()["shared_count"]) | |
| self.assertEqual(self.kb.bases({"tenant_id": "another", "user_id": "u-lin"})["items"], []) | |
| def test_seed_retrieval_cases_top3(self): | |
| mapping = {"archive_matter_a": {"principal": A, "workspace_id": self.wa}, | |
| "inventory_matter_b": {"principal": B, "workspace_id": self.wb}} | |
| self.assertEqual(len(self.kb.import_seeds(mapping)), 20) | |
| cases = json.loads((ROOT / "docs/05_知识库种子素材/retrieval_cases.json").read_text()) | |
| for case in cases[:5]: | |
| with self.subTest(case=case["id"]): | |
| user, wid = (A, self.wa) if case["user"] == "demo_a" else (B, self.wb) | |
| hits = self.search(case["query"], user, wid) | |
| self.assertTrue(set(case["expected_top3_any"]) & {x["document_id"] for x in hits}) | |
| if "expected_fact" in case: | |
| self.assertTrue(any(case["expected_fact"] in h["snippet"] for h in hits)) | |
| self.assertEqual(self.search("B_PRIVATE_MARKER"), []) | |
| self.assertEqual(self.search("A_PRIVATE_MARKER", B, self.wb), []) | |
| other = self.ws.create(A, "另一归档事项", "another-seed-matter")[0]["workspace_id"] | |
| self.assertEqual(self.search("A_PRIVATE_MARKER", A, other), []) | |
| private_b = self.search("B_PRIVATE_MARKER", B, self.wb)[0] | |
| with self.assertRaises(KnowledgeError): | |
| self.kb.citation(A, private_b["citation_id"], self.wa) | |
| def test_attachment_private_scope_and_citation(self): | |
| self.ingest() | |
| hit = self.search("PRIVATE_A")[0] | |
| self.assertIn("2026年10月20日", self.kb.citation(A, hit["citation_id"], self.wa)["text"]) | |
| other = self.ws.create(A, "另一事项", "other-matter")[0]["workspace_id"] | |
| self.assertEqual(self.search("PRIVATE_A", A, other), []) | |
| self.assertEqual(self.search("PRIVATE_A", B, self.wb), []) | |
| for user, wid in [(B, self.wb), (A, other), (A, None)]: | |
| with self.assertRaises(KnowledgeError) as ctx: | |
| self.kb.citation(user, hit["citation_id"], wid) | |
| self.assertEqual(ctx.exception.status, 404) | |
| def test_explicit_denied_base_and_cross_user_workspace(self): | |
| with self.assertRaises(KnowledgeError): | |
| self.search("资料", bases=["unknown"]) | |
| with self.assertRaises(Exception) as ctx: | |
| self.search("资料", B, self.wa) | |
| self.assertEqual(ctx.exception.status, 404) | |
| def test_document_source_preserves_full_original_without_chunk_overlap(self): | |
| text='完整文件验收\r\n'+''.join(f'会议记录第{i}行:责任人核对任务。\r\n' for i in range(1300))+'原文件结尾' | |
| self.ingest(name='完整会议记录.txt',content=text) | |
| citation=self.search('完整文件验收')[0] | |
| self.assertNotIn('原文件结尾',self.kb.citation(A,citation['citation_id'],self.wa)['text']) | |
| source=self.kb.document_source(A,citation['citation_id'],self.wa) | |
| self.assertEqual(source['text'],text) | |
| self.assertEqual(source['filename'],'完整会议记录.txt') | |
| self.assertEqual(source['preview_scope'],'document') | |
| filename,content=self.kb.original(A,citation['citation_id'],self.wa) | |
| self.assertEqual(content,text.encode()) | |
| self.assertEqual(filename,source['filename']) | |
| def test_document_source_does_not_bypass_personal_workspace_permissions(self): | |
| self.ingest() | |
| citation=self.search('PRIVATE_A')[0] | |
| other=self.ws.create(A,'其他事项','source-other')[0]['workspace_id'] | |
| for user,wid in [(B,self.wb),(A,other),(A,None)]: | |
| with self.assertRaises(KnowledgeError) as denied: | |
| self.kb.document_source(user,citation['citation_id'],wid) | |
| self.assertEqual(denied.exception.status,404) | |
| def test_empty_or_unmatched_query(self): | |
| self.assertEqual(self.search("ABSENT_SECRET_MARKER"), []) | |
| with self.assertRaises(KnowledgeError): | |
| self.search(" ") | |
| def test_grant_revocation_survives_restart_and_denies_citation(self): | |
| hit = self.search("资料归档", bases=["kb-rules"])[0] | |
| with self.ws.Session.begin() as db: | |
| db.execute(delete(Grant).where(Grant.tenant == A["tenant_id"], Grant.base_id == "kb-rules", Grant.user_id == A["user_id"])) | |
| self.kb = KnowledgeService(self.ws) | |
| with self.assertRaises(KnowledgeError): | |
| self.kb.citation(A, hit["citation_id"]) | |
| with self.assertRaises(KnowledgeError): | |
| self.search("资料归档", bases=["kb-rules"]) | |
| def test_expired_documents_excluded_from_search_and_citation(self): | |
| hit = self.search("资料归档", bases=["kb-rules"])[0] | |
| with self.ws.Session.begin() as db: | |
| doc = db.scalar(select(Document).where(Document.document_id == hit["document_id"])) | |
| doc.valid_to = "2000-01-01" | |
| with self.assertRaises(KnowledgeError): | |
| self.kb.citation(A, hit["citation_id"]) | |
| self.assertNotIn(hit["document_id"], {x["document_id"] for x in self.search("资料归档")}) | |
| def test_ingestion_restart_and_replay_have_stable_ids(self): | |
| a = self.ingest() | |
| hit = self.search("PRIVATE_A")[0] | |
| self.kb = KnowledgeService(self.ws) | |
| self.kb.consume_event({"event_type": "document.ingest.requested", "tenant_id": A["tenant_id"], | |
| "owner_user_id": A["user_id"], "workspace_id": self.wa, "payload": {"attachment_id": a["attachment_id"]}}) | |
| self.assertEqual(self.search("PRIVATE_A")[0]["citation_id"], hit["citation_id"]) | |
| with self.ws.Session() as db: | |
| self.assertEqual(db.scalar(select(func.count()).select_from(Document).where(Document.owner == A["user_id"])), 1) | |
| def test_changed_document_version_rejected(self): | |
| kwargs = dict(tenant="demo-tenant", document_id="immutable", version="1", base_id="kb-rules", title="Title", filename="doc.md") | |
| self.kb._save(**kwargs, content=b"first") | |
| with self.assertRaises(KnowledgeError) as ctx: | |
| self.kb._save(**kwargs, content=b"second") | |
| self.assertEqual(ctx.exception.code, "DOCUMENT_VERSION_CONFLICT") | |
| def test_latest_version_search_and_historical_citation(self): | |
| kwargs = dict(tenant="demo-tenant", document_id="versions", base_id="kb-rules", title="Title", filename="doc.md") | |
| self.kb._save(**kwargs, version="1", content=b"OLD_VERSION_MARKER") | |
| old = self.search("OLD_VERSION_MARKER")[0] | |
| self.kb._save(**kwargs, version="2", content=b"NEW_VERSION_MARKER") | |
| self.assertEqual(self.search("OLD_VERSION_MARKER"), []) | |
| self.assertEqual(self.search("NEW_VERSION_MARKER")[0]["document_version"], 2) | |
| self.assertIn("OLD_VERSION_MARKER", self.kb.citation(A, old["citation_id"])["text"]) | |
| def test_failed_pdf_is_recorded_and_not_retrievable(self): | |
| self.ingest(name="broken.pdf", content=b"%PDF-1.7\ninvalid") | |
| attachment = self.ws.attachments(A, self.wa)["items"][0] | |
| self.assertEqual(attachment["parse_status"], "failed") | |
| docs = [x for x in self.kb.documents(A, self.wa)["items"] if x["source_type"] == "attachment"] | |
| self.assertEqual(docs[0]["error_code"], "DOCUMENT_PARSE_FAILED") | |
| def test_storage_failure_keeps_outbox_pending_then_recovers(self): | |
| a, _ = self.ws.attachment(A, self.wa, "retry.md", b"RETRY_MARKER", "retry-key") | |
| original = self.ws.attachment_content | |
| self.ws.attachment_content = lambda *args: (_ for _ in ()).throw(OSError("temporary")) | |
| self.assertEqual(self.ws.dispatch_pending(self.kb.consume_event, event_types={"document.ingest.requested"}), 0) | |
| self.ws.attachment_content = original | |
| self.assertEqual(self.ws.dispatch_pending(self.kb.consume_event, event_types={"document.ingest.requested"}), 1) | |
| self.assertTrue(self.search("RETRY_MARKER")) | |
| def test_docx_paragraph_and_table_text(self): | |
| stream = io.BytesIO() | |
| xml = '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>DOCX_PRIVATE</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>表格来文</w:t></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>' | |
| with zipfile.ZipFile(stream, "w") as archive: | |
| archive.writestr("word/document.xml", xml) | |
| self.ingest(name="sample.docx", content=stream.getvalue()) | |
| self.assertIn("段落 1", self.search("DOCX_PRIVATE")[0]["locator"]) | |
| self.assertTrue(self.search("表格来文")) | |
| def test_docx_entities_and_expansion_rejected(self): | |
| for xml in ['<!DOCTYPE x [<!ENTITY secret SYSTEM "file:///etc/passwd">]><x>&secret;</x>', "x" * (20 * 1024 * 1024 + 1)]: | |
| stream = io.BytesIO() | |
| with zipfile.ZipFile(stream, "w", zipfile.ZIP_DEFLATED) as archive: | |
| archive.writestr("word/document.xml", xml) | |
| with self.assertRaises(ParseError): | |
| extract("sample.docx", stream.getvalue()) | |
| def test_pdf_text_page_and_scanned_failure(self): | |
| from pypdf import PdfWriter | |
| from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject | |
| writer = PdfWriter() | |
| page = writer.add_blank_page(300, 300) | |
| font = DictionaryObject({NameObject("/Type"): NameObject("/Font"), NameObject("/Subtype"): NameObject("/Type1"), NameObject("/BaseFont"): NameObject("/Helvetica")}) | |
| page[NameObject("/Resources")] = DictionaryObject({NameObject("/Font"): DictionaryObject({NameObject("/F1"): font})}) | |
| stream = DecodedStreamObject(); stream.set_data(b"BT /F1 12 Tf 10 100 Td (PDF_PRIVATE) Tj ET") | |
| page[NameObject("/Contents")] = writer._add_object(stream) | |
| output = io.BytesIO(); writer.write(output) | |
| self.ingest(name="text.pdf", content=output.getvalue()) | |
| self.assertIn("第 1 页", self.search("PDF_PRIVATE")[0]["locator"]) | |
| blank = PdfWriter(); blank.add_blank_page(100, 100) | |
| output = io.BytesIO(); blank.write(output) | |
| with self.assertRaisesRegex(ParseError, "NO_EXTRACTABLE_TEXT"): | |
| extract("scan.pdf", output.getvalue()) | |
| if __name__ == "__main__": | |
| unittest.main() | |