"""Single-process SQL Knowledge Port, sharing an engine but owning kb_* tables.""" from __future__ import annotations from observability import traced, span import hashlib import json import os import threading from datetime import datetime, timezone from pathlib import Path from sqlalchemy import and_, or_, select from .models import Base, KnowledgeBase, Grant, Document, Chunk, KnowledgeError from .parsing import ParseError, extract, extract_units, rank from .management import Management from .models import LibraryConfig,DocumentInfo,DocumentSource BASES = [ ("kb-rules", "制度与工作依据库", "shared"), ("kb-history", "历史公文案例库", "shared"), ("kb-templates", "公文模板与要素库", "shared"), ("kb-circulation", "发文与办理流程库", "shared"), ("kb-personal", "个人来文与事项材料库", "personal"), ("kb-office", "邮件与会议知识库", "shared"), ] ALIASES = {**{f"KB0{i+1}": row[0] for i, row in enumerate(BASES)}, "kb-policy": "kb-rules"} SEEDS = Path(__file__).resolve().parents[1] / "docs" / "05_知识库种子素材" def digest(value): return hashlib.sha256(value.encode("utf-8")).hexdigest() def missing(): return KnowledgeError(404, "RESOURCE_NOT_FOUND", "资源不存在或无权访问。") class KnowledgeService(Management): mode = "sql-bm25-persistent" def __init__(self, workspace, seed_demo=None): self.workspace = workspace self.Session = workspace.Session self.lock = threading.RLock() if os.getenv("WORKOS_WORKSPACE_AUTO_SCHEMA", "1") == "1": Base.metadata.create_all(workspace.engine) should_seed = seed_demo if seed_demo is not None else os.getenv("WORKOS_KNOWLEDGE_SEED_DEMO", "1") == "1" if should_seed: self.seed_demo_catalog() def seed_demo_catalog(self): """Grant demo users explicitly; never infer grants for other tenants/users.""" from .seed import corpus_plan, import_catalog plan=corpus_plan() # No partial import for a corrupt bundled file. with self.lock, self.Session.begin() as db: for bid, name, visibility in BASES: if db.get(KnowledgeBase, ("demo-tenant", bid)) is None: db.add(KnowledgeBase(tenant="demo-tenant", id=bid, name=name, visibility=visibility)) for uid in ("u-lin", "u-zhou", "u-admin"): db.add(Grant(tenant="demo-tenant", base_id=bid, user_id=uid)) if db.get(LibraryConfig,('demo-tenant',bid)) is None: category={'kb-rules':'制度依据','kb-history':'写作案例','kb-templates':'公文模板','kb-circulation':'办理流程','kb-personal':'事项附件','kb-office':'邮件与会议'}[bid] db.add(LibraryConfig(tenant='demo-tenant',base_id=bid,description=name+':可核对出处的办公合成样例。' if visibility=='shared' else '本人当前事项上传的来文与会议转写。',category=category,status='enabled',revision=1)) import_catalog(self, plan) def _bases(self, db, user): rows = list(db.scalars(select(KnowledgeBase).join(Grant, and_( Grant.tenant == KnowledgeBase.tenant, Grant.base_id == KnowledgeBase.id)).where( KnowledgeBase.tenant == user["tenant_id"], Grant.user_id == user["user_id"]))) if hasattr(self,"organization"): extra=self.organization.allowed_bases(user) rows=list({b.id:b for b in rows+list(db.scalars(select(KnowledgeBase).where(KnowledgeBase.tenant==user["tenant_id"],KnowledgeBase.id.in_(extra))))}.values()) return [b for b in rows if (db.get(LibraryConfig, (b.tenant, b.id)) is None or db.get(LibraryConfig, (b.tenant, b.id)).status == "enabled")] def bases(self, user): return self.library_list(user) def _scope(self, db, user, wid, requested=None): allowed = {b.id: b for b in self._bases(db, user)} ids = list(dict.fromkeys(ALIASES.get(b, b) for b in (requested or allowed))) if any(b not in allowed for b in ids): raise missing() shared = [bid for bid in ids if allowed[bid].visibility == "shared"] personal = [bid for bid in ids if allowed[bid].visibility == "personal"] return and_(Document.tenant == user["tenant_id"], or_( and_(Document.base_id.in_(shared), Document.owner.is_(None), Document.workspace.is_(None)), and_(Document.base_id.in_(personal), Document.owner == user["user_id"], Document.workspace == wid))) @staticmethod def _valid(): today = datetime.now(timezone.utc).date().isoformat() return and_(Document.status == "ready", or_(Document.valid_from.is_(None), Document.valid_from <= today), or_(Document.valid_to.is_(None), Document.valid_to >= today)) def _citation(self, chunk, doc, score=None, full=False, sources=None): result = {"citation_id": chunk.id, "chunk_id": chunk.id, "document_id": doc.document_id, "document_version": int(doc.version) if doc.version.isdigit() else doc.version, "base_id": doc.base_id, "title": doc.title, "locator": chunk.locator, "snippet": chunk.text[:800], "content_hash": doc.sha256, "source_type": doc.source_type, "provider_mode": "persistent"} if sources is not None: if sources.get(doc.id):result["provenance"]=sources[doc.id] else: with self.Session() as db: source=db.get(DocumentSource,doc.id) if source:result["provenance"]=source.body if score is not None: result["score"] = round(score, 6) if full: result["text"] = chunk.text return result @traced("Knowledge Service") def search(self, user, wid, query, base_ids, top_k): self.workspace.assert_access(user, wid) if not isinstance(query, str) or not query.strip() or len(query) > 2000 or not 1 <= top_k <= 10: raise KnowledgeError(422, "INVALID_SEARCH", "检索问题和条数不符合约束。") with self.Session() as db: rows = db.execute(select(Chunk, Document).join(Document).outerjoin(DocumentInfo,DocumentInfo.document_key==Document.id).where( self._scope(db, user, wid, base_ids), self._valid(),or_(DocumentInfo.document_key.is_(None),DocumentInfo.publication=='published'))).all() # Use the most recently imported valid immutable version in this authorized scope. latest = {} for _, doc in rows: marker = (doc.created_at, doc.id) if marker > latest.get(doc.document_id, ("", "")): latest[doc.document_id] = marker rows = [(c, d) for c, d in rows if (d.created_at, d.id) == latest[d.document_id]] sources={s.document_key:s.body for s in db.scalars(select(DocumentSource).where(DocumentSource.document_key.in_({d.id for c,d in rows})))} with span('Knowledge Recall','authorized current chunks',{'base_ids':base_ids,'query':query}) as stage: stage['value']={'authorized_chunk_count':len(rows),'authorized_document_count':len({d.id for c,d in rows}),'filter':'tenant + user/group grant + personal owner/workspace + published + effective dates + latest version'} ranked=rank(query,rows,max(1,len(rows))) picked={c.id for score,c,d in ranked[:top_k]} scores={c.id:score for score,c,d in ranked} # Persist a full authorized chunk ledger; details are paginated independently of span caps. from observability import chunk_ledger ledger=[{**self._citation(c,d,scores.get(c.id,0),sources=sources), 'ordinal':c.ordinal,'decision':'selected' if c.id in picked else 'below_top_k' if c.id in scores else 'no_term_match'} for c,d in rows] chunk_ledger(ledger) with span('Knowledge Recall','select ranked chunks',{'algorithm':'BM25','top_k':top_k}) as stage: stage['value']={'candidate_count':len(rows),'positive_score_count':len(ranked),'selected':[self._citation(c,d,score,sources=sources) for score,c,d in ranked[:top_k]],'chunk_ledger_count':len(ledger)} return {"items": [self._citation(c, d, score,sources=sources) for score, c, d in ranked[:top_k]], "next_cursor": None, "mode": self.mode} @traced("Knowledge Service") def citation(self, user, cid, workspace_id=None): if workspace_id: self.workspace.assert_access(user, workspace_id) with self.Session() as db: # A private citation needs both the owner and current workspace. IDs are not capabilities. row = db.execute(select(Chunk, Document).join(Document).where( Chunk.id == cid, self._scope(db, user, workspace_id), self._valid())).first() if row is None or not self._published(db, row[1]): raise missing() return self._citation(*row, full=True) def document_source(self, user, cid, workspace_id=None): citation = self.citation(user, cid, workspace_id) filename, content = self.original(user, cid, workspace_id) try: text = '\n\n'.join(text for _, text in extract_units(filename, content)) except ParseError: raise KnowledgeError(422, 'ORIGINAL_PREVIEW_UNAVAILABLE', '原文件暂不支持文字预览,请下载查看。') from None return {**citation, 'filename': filename, 'text': text, 'locator': '完整原文件', 'preview_scope': 'document'} def documents(self, user, wid): self.workspace.assert_access(user, wid) with self.Session() as db: rows = db.scalars(select(Document).where(self._scope(db, user, wid)).order_by(Document.id)).all() return {"items": [{"document_id": d.document_id, "document_version": d.version, "title": d.title, "base_id": d.base_id, "status": d.status, "error_code": d.error_code, "source_type": d.source_type, "content_hash": d.sha256} for d in rows], "next_cursor": None} @traced('Knowledge Service','read current meeting attachments') def meeting_materials(self,user,wid): self.workspace.assert_access(user,wid) with self.Session() as db: rows=db.execute(select(Chunk,Document).join(Document).where(self._scope(db,user,wid,['kb-personal']),self._valid()).order_by(Document.created_at,Chunk.ordinal)).all() rows=[(c,d) for c,d in rows if self._published(db,d)] latest={} for _,d in rows:latest[d.document_id]=max(latest.get(d.document_id,('', '')),(d.created_at,d.id)) rows=[(c,d) for c,d in rows if latest[d.document_id]==(d.created_at,d.id)] if sum(len(c.text) for c,_ in rows)>32000: raise KnowledgeError(413,'MEETING_MATERIAL_TOO_LONG','当前事项转写超过32000字,请按议题拆分到不同事项后整理。') return {'items':[self._citation(c,d) for c,d in rows]} @traced("Knowledge Service") def _save(self, *, tenant, document_id, version, base_id, title, content, filename, owner=None, workspace=None, source_type="attachment", valid_from=None, valid_to=None, provenance=None): """Atomic immutable version+chunks. Same identity with changed bytes is a conflict.""" sha = hashlib.sha256(content).hexdigest() key = "doc-" + digest(json.dumps([tenant, document_id, version])) try: chunks, error = extract(filename, content), None except ParseError as exc: chunks, error = [], str(exc) with self.lock, self.Session.begin() as db: base = db.get(KnowledgeBase, (tenant, base_id)) if base is None or (base.visibility == "personal" and (not owner or not workspace)): raise missing() if base.visibility == "shared" and (owner is not None or workspace is not None): raise KnowledgeError(422, "INVALID_DOCUMENT_SCOPE", "共享库不能存放私人事项材料。") old = db.get(Document, key) previous = db.scalar(select(Document).where(Document.tenant == tenant, Document.document_id == document_id)) if previous and (previous.base_id, previous.owner, previous.workspace) != (base_id, owner, workspace): raise KnowledgeError(409, "DOCUMENT_SCOPE_CONFLICT", "文档新版本不能改变授权归属。") if old is not None: if (old.sha256, old.base_id, old.owner, old.workspace) != (sha, base_id, owner, workspace): raise KnowledgeError(409, "DOCUMENT_VERSION_CONFLICT", "同一文档版本不可替换内容或归属。") self._info(db, old, filename, content, chunks) if provenance is not None: source=db.get(DocumentSource,key) if source:source.body={**provenance,"content_sha256":sha,"sha256":sha} else:db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha})) return old doc = Document(id=key, tenant=tenant, document_id=document_id, version=version, base_id=base_id, title=title, owner=owner, workspace=workspace, source_type=source_type, sha256=sha, status="failed" if error else "ready", error_code=error, valid_from=valid_from, valid_to=valid_to, created_at=datetime.now(timezone.utc).isoformat()) db.add(doc) db.flush() self._info(db, doc, filename, content, chunks) if provenance is not None: db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha})) for ordinal, (locator, text, terms) in enumerate(chunks, 1): db.add(Chunk(id="cite-" + digest(key + ":" + str(ordinal)), document_key=key, ordinal=ordinal, locator=locator, text=text, terms=terms)) return doc @traced("Knowledge Service") def consume_event(self, envelope): if envelope["event_type"] != "document.ingest.requested": return False user = {"tenant_id": envelope["tenant_id"], "user_id": envelope["owner_user_id"]} aid, wid = envelope["payload"]["attachment_id"], envelope["workspace_id"] info = self.workspace.attachment_info(user, aid) if info["workspace_id"] != wid: raise missing() with self.Session() as db: if "kb-personal" not in {b.id for b in self._bases(db, user)}: raise missing() doc = self._save(tenant=user["tenant_id"], document_id=aid, version="1", base_id="kb-personal", title=info["filename"], filename=info["filename"], content=self.workspace.attachment_content(user, aid), owner=user["user_id"], workspace=wid) # DB/storage exceptions propagate so the durable outbox remains pending for retry. self.workspace.apply_ingest_result(user, aid, doc.status, doc.document_id, doc.error_code) return True def import_seeds(self, private_mapping=None): """Trusted local operator entry: private_mapping maps demo key to real principal+workspace.""" manifest = json.loads((SEEDS / "manifest.json").read_text(encoding="utf-8-sig")) imported = [] for meta in manifest["documents"]: owner, wid = None, None if meta["owner_user_id"]: mapping = (private_mapping or {}).get(meta["workspace_key"]) if mapping is None: continue user, wid = mapping["principal"], mapping["workspace_id"] self.workspace.assert_access(user, wid) if user["tenant_id"] != "demo-tenant": raise missing() owner = user["user_id"] content = (SEEDS / meta["file"]).read_bytes().replace(b"\r\n", b"\n") if hashlib.sha256(content).hexdigest() != meta["sha256"]: raise KnowledgeError(422, "SEED_HASH_MISMATCH", "种子资料校验失败。") doc = self._save(tenant="demo-tenant", document_id=meta["document_id"], version=meta["version"], base_id=ALIASES[meta["kb_id"]], title=meta["title"], content=content, filename=meta["file"], owner=owner, workspace=wid, source_type=meta["source_type"], valid_from=meta["valid_from"], valid_to=meta["valid_to"]) imported.append({"document_id": doc.document_id, "status": doc.status}) return imported