"""Run from base1: python -m unittest discover -s KnowledgeService/tests -v.""" import io import json import os import sys import tempfile import unittest import zipfile from pathlib import Path from sqlalchemy import delete, func, select ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "APIGateway")) from WorkspaceService.workspace_service import WorkspaceService from KnowledgeService import KnowledgeService, KnowledgeError from KnowledgeService.models import Chunk, Document, Grant from KnowledgeService.parsing import ParseError, extract A = {"tenant_id": "demo-tenant", "user_id": "u-lin"} B = {"tenant_id": "demo-tenant", "user_id": "u-zhou"} class KnowledgeTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.ws = WorkspaceService(database_url="sqlite:///" + self.tmp.name + "/test.db", file_root=self.tmp.name + "/files", auto_create_schema=True) self.kb = KnowledgeService(self.ws) self.wa = self.ws.create(A, "归档事项", "matter-a")[0]["workspace_id"] self.wb = self.ws.create(B, "设备盘点", "matter-b")[0]["workspace_id"] def tearDown(self): self.ws.engine.dispose() self.tmp.cleanup() def ingest(self, user=A, wid=None, name="notice.md", content="资料归档截止日期 PRIVATE_A 2026年10月20日"): wid = wid or self.wa a, _ = self.ws.attachment(user, wid, name, content.encode() if isinstance(content, str) else content, name) sent = self.ws.dispatch_pending(self.kb.consume_event, event_types={"document.ingest.requested"}) self.assertEqual(sent, 1) return a def search(self, query, user=A, wid=None, bases=None): return self.kb.search(user, wid or self.wa, query, bases or [], 3)["items"] def test_five_shared_foundations_and_private_base(self): bases = self.kb.bases(A)["items"] self.assertEqual(len([b for b in bases if b["visibility"] == "shared"]), 5) self.assertEqual(len([b for b in bases if b["visibility"] == "personal"]), 1) from KnowledgeService.seed import inventory self.assertEqual(len(self.kb.documents(A, self.wa)["items"]), inventory()["shared_count"]) self.assertEqual(self.kb.bases({"tenant_id": "another", "user_id": "u-lin"})["items"], []) def test_seed_retrieval_cases_top3(self): mapping = {"archive_matter_a": {"principal": A, "workspace_id": self.wa}, "inventory_matter_b": {"principal": B, "workspace_id": self.wb}} self.assertEqual(len(self.kb.import_seeds(mapping)), 20) cases = json.loads((ROOT / "docs/05_知识库种子素材/retrieval_cases.json").read_text()) for case in cases[:5]: with self.subTest(case=case["id"]): user, wid = (A, self.wa) if case["user"] == "demo_a" else (B, self.wb) hits = self.search(case["query"], user, wid) self.assertTrue(set(case["expected_top3_any"]) & {x["document_id"] for x in hits}) if "expected_fact" in case: self.assertTrue(any(case["expected_fact"] in h["snippet"] for h in hits)) self.assertEqual(self.search("B_PRIVATE_MARKER"), []) self.assertEqual(self.search("A_PRIVATE_MARKER", B, self.wb), []) other = self.ws.create(A, "另一归档事项", "another-seed-matter")[0]["workspace_id"] self.assertEqual(self.search("A_PRIVATE_MARKER", A, other), []) private_b = self.search("B_PRIVATE_MARKER", B, self.wb)[0] with self.assertRaises(KnowledgeError): self.kb.citation(A, private_b["citation_id"], self.wa) def test_attachment_private_scope_and_citation(self): self.ingest() hit = self.search("PRIVATE_A")[0] self.assertIn("2026年10月20日", self.kb.citation(A, hit["citation_id"], self.wa)["text"]) other = self.ws.create(A, "另一事项", "other-matter")[0]["workspace_id"] self.assertEqual(self.search("PRIVATE_A", A, other), []) self.assertEqual(self.search("PRIVATE_A", B, self.wb), []) for user, wid in [(B, self.wb), (A, other), (A, None)]: with self.assertRaises(KnowledgeError) as ctx: self.kb.citation(user, hit["citation_id"], wid) self.assertEqual(ctx.exception.status, 404) def test_explicit_denied_base_and_cross_user_workspace(self): with self.assertRaises(KnowledgeError): self.search("资料", bases=["unknown"]) with self.assertRaises(Exception) as ctx: self.search("资料", B, self.wa) self.assertEqual(ctx.exception.status, 404) def test_document_source_preserves_full_original_without_chunk_overlap(self): text='完整文件验收\r\n'+''.join(f'会议记录第{i}行:责任人核对任务。\r\n' for i in range(1300))+'原文件结尾' self.ingest(name='完整会议记录.txt',content=text) citation=self.search('完整文件验收')[0] self.assertNotIn('原文件结尾',self.kb.citation(A,citation['citation_id'],self.wa)['text']) source=self.kb.document_source(A,citation['citation_id'],self.wa) self.assertEqual(source['text'],text) self.assertEqual(source['filename'],'完整会议记录.txt') self.assertEqual(source['preview_scope'],'document') filename,content=self.kb.original(A,citation['citation_id'],self.wa) self.assertEqual(content,text.encode()) self.assertEqual(filename,source['filename']) def test_document_source_does_not_bypass_personal_workspace_permissions(self): self.ingest() citation=self.search('PRIVATE_A')[0] other=self.ws.create(A,'其他事项','source-other')[0]['workspace_id'] for user,wid in [(B,self.wb),(A,other),(A,None)]: with self.assertRaises(KnowledgeError) as denied: self.kb.document_source(user,citation['citation_id'],wid) self.assertEqual(denied.exception.status,404) def test_empty_or_unmatched_query(self): self.assertEqual(self.search("ABSENT_SECRET_MARKER"), []) with self.assertRaises(KnowledgeError): self.search(" ") def test_grant_revocation_survives_restart_and_denies_citation(self): hit = self.search("资料归档", bases=["kb-rules"])[0] with self.ws.Session.begin() as db: db.execute(delete(Grant).where(Grant.tenant == A["tenant_id"], Grant.base_id == "kb-rules", Grant.user_id == A["user_id"])) self.kb = KnowledgeService(self.ws) with self.assertRaises(KnowledgeError): self.kb.citation(A, hit["citation_id"]) with self.assertRaises(KnowledgeError): self.search("资料归档", bases=["kb-rules"]) def test_expired_documents_excluded_from_search_and_citation(self): hit = self.search("资料归档", bases=["kb-rules"])[0] with self.ws.Session.begin() as db: doc = db.scalar(select(Document).where(Document.document_id == hit["document_id"])) doc.valid_to = "2000-01-01" with self.assertRaises(KnowledgeError): self.kb.citation(A, hit["citation_id"]) self.assertNotIn(hit["document_id"], {x["document_id"] for x in self.search("资料归档")}) def test_ingestion_restart_and_replay_have_stable_ids(self): a = self.ingest() hit = self.search("PRIVATE_A")[0] self.kb = KnowledgeService(self.ws) self.kb.consume_event({"event_type": "document.ingest.requested", "tenant_id": A["tenant_id"], "owner_user_id": A["user_id"], "workspace_id": self.wa, "payload": {"attachment_id": a["attachment_id"]}}) self.assertEqual(self.search("PRIVATE_A")[0]["citation_id"], hit["citation_id"]) with self.ws.Session() as db: self.assertEqual(db.scalar(select(func.count()).select_from(Document).where(Document.owner == A["user_id"])), 1) def test_changed_document_version_rejected(self): kwargs = dict(tenant="demo-tenant", document_id="immutable", version="1", base_id="kb-rules", title="Title", filename="doc.md") self.kb._save(**kwargs, content=b"first") with self.assertRaises(KnowledgeError) as ctx: self.kb._save(**kwargs, content=b"second") self.assertEqual(ctx.exception.code, "DOCUMENT_VERSION_CONFLICT") def test_latest_version_search_and_historical_citation(self): kwargs = dict(tenant="demo-tenant", document_id="versions", base_id="kb-rules", title="Title", filename="doc.md") self.kb._save(**kwargs, version="1", content=b"OLD_VERSION_MARKER") old = self.search("OLD_VERSION_MARKER")[0] self.kb._save(**kwargs, version="2", content=b"NEW_VERSION_MARKER") self.assertEqual(self.search("OLD_VERSION_MARKER"), []) self.assertEqual(self.search("NEW_VERSION_MARKER")[0]["document_version"], 2) self.assertIn("OLD_VERSION_MARKER", self.kb.citation(A, old["citation_id"])["text"]) def test_failed_pdf_is_recorded_and_not_retrievable(self): self.ingest(name="broken.pdf", content=b"%PDF-1.7\ninvalid") attachment = self.ws.attachments(A, self.wa)["items"][0] self.assertEqual(attachment["parse_status"], "failed") docs = [x for x in self.kb.documents(A, self.wa)["items"] if x["source_type"] == "attachment"] self.assertEqual(docs[0]["error_code"], "DOCUMENT_PARSE_FAILED") def test_storage_failure_keeps_outbox_pending_then_recovers(self): a, _ = self.ws.attachment(A, self.wa, "retry.md", b"RETRY_MARKER", "retry-key") original = self.ws.attachment_content self.ws.attachment_content = lambda *args: (_ for _ in ()).throw(OSError("temporary")) self.assertEqual(self.ws.dispatch_pending(self.kb.consume_event, event_types={"document.ingest.requested"}), 0) self.ws.attachment_content = original self.assertEqual(self.ws.dispatch_pending(self.kb.consume_event, event_types={"document.ingest.requested"}), 1) self.assertTrue(self.search("RETRY_MARKER")) def test_docx_paragraph_and_table_text(self): stream = io.BytesIO() xml = 'DOCX_PRIVATE表格来文' with zipfile.ZipFile(stream, "w") as archive: archive.writestr("word/document.xml", xml) self.ingest(name="sample.docx", content=stream.getvalue()) self.assertIn("段落 1", self.search("DOCX_PRIVATE")[0]["locator"]) self.assertTrue(self.search("表格来文")) def test_docx_entities_and_expansion_rejected(self): for xml in [']>&secret;', "x" * (20 * 1024 * 1024 + 1)]: stream = io.BytesIO() with zipfile.ZipFile(stream, "w", zipfile.ZIP_DEFLATED) as archive: archive.writestr("word/document.xml", xml) with self.assertRaises(ParseError): extract("sample.docx", stream.getvalue()) def test_pdf_text_page_and_scanned_failure(self): from pypdf import PdfWriter from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject writer = PdfWriter() page = writer.add_blank_page(300, 300) font = DictionaryObject({NameObject("/Type"): NameObject("/Font"), NameObject("/Subtype"): NameObject("/Type1"), NameObject("/BaseFont"): NameObject("/Helvetica")}) page[NameObject("/Resources")] = DictionaryObject({NameObject("/Font"): DictionaryObject({NameObject("/F1"): font})}) stream = DecodedStreamObject(); stream.set_data(b"BT /F1 12 Tf 10 100 Td (PDF_PRIVATE) Tj ET") page[NameObject("/Contents")] = writer._add_object(stream) output = io.BytesIO(); writer.write(output) self.ingest(name="text.pdf", content=output.getvalue()) self.assertIn("第 1 页", self.search("PDF_PRIVATE")[0]["locator"]) blank = PdfWriter(); blank.add_blank_page(100, 100) output = io.BytesIO(); blank.write(output) with self.assertRaisesRegex(ParseError, "NO_EXTRACTABLE_TEXT"): extract("scan.pdf", output.getvalue()) if __name__ == "__main__": unittest.main()