dellweb3's picture
Group original file references and export generated meeting Markdown (part 2)
67e287e verified
Raw History Blame Contribute Delete
17 kB
"""Single-process SQL Knowledge Port, sharing an engine but owning kb_* tables."""
from __future__ import annotations
from observability import traced, span
import hashlib
import json
import os
import threading
from datetime import datetime, timezone
from pathlib import Path
from sqlalchemy import and_, or_, select
from .models import Base, KnowledgeBase, Grant, Document, Chunk, KnowledgeError
from .parsing import ParseError, extract, extract_units, rank
from .management import Management
from .models import LibraryConfig,DocumentInfo,DocumentSource
BASES = [
("kb-rules", "制度与工作依据库", "shared"),
("kb-history", "历史公文案例库", "shared"),
("kb-templates", "公文模板与要素库", "shared"),
("kb-circulation", "发文与办理流程库", "shared"),
("kb-personal", "个人来文与事项材料库", "personal"),
("kb-office", "邮件与会议知识库", "shared"),
]
ALIASES = {**{f"KB0{i+1}": row[0] for i, row in enumerate(BASES)}, "kb-policy": "kb-rules"}
SEEDS = Path(__file__).resolve().parents[1] / "docs" / "05_知识库种子素材"
def digest(value):
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def missing():
return KnowledgeError(404, "RESOURCE_NOT_FOUND", "资源不存在或无权访问。")
class KnowledgeService(Management):
mode = "sql-bm25-persistent"
def __init__(self, workspace, seed_demo=None):
self.workspace = workspace
self.Session = workspace.Session
self.lock = threading.RLock()
if os.getenv("WORKOS_WORKSPACE_AUTO_SCHEMA", "1") == "1":
Base.metadata.create_all(workspace.engine)
should_seed = seed_demo if seed_demo is not None else os.getenv("WORKOS_KNOWLEDGE_SEED_DEMO", "1") == "1"
if should_seed:
self.seed_demo_catalog()
def seed_demo_catalog(self):
"""Grant demo users explicitly; never infer grants for other tenants/users."""
from .seed import corpus_plan, import_catalog
plan=corpus_plan() # No partial import for a corrupt bundled file.
with self.lock, self.Session.begin() as db:
for bid, name, visibility in BASES:
if db.get(KnowledgeBase, ("demo-tenant", bid)) is None:
db.add(KnowledgeBase(tenant="demo-tenant", id=bid, name=name, visibility=visibility))
for uid in ("u-lin", "u-zhou", "u-admin"):
db.add(Grant(tenant="demo-tenant", base_id=bid, user_id=uid))
if db.get(LibraryConfig,('demo-tenant',bid)) is None:
category={'kb-rules':'制度依据','kb-history':'写作案例','kb-templates':'公文模板','kb-circulation':'办理流程','kb-personal':'事项附件','kb-office':'邮件与会议'}[bid]
db.add(LibraryConfig(tenant='demo-tenant',base_id=bid,description=name+':可核对出处的办公合成样例。' if visibility=='shared' else '本人当前事项上传的来文与会议转写。',category=category,status='enabled',revision=1))
import_catalog(self, plan)
def _bases(self, db, user):
rows = list(db.scalars(select(KnowledgeBase).join(Grant, and_(
Grant.tenant == KnowledgeBase.tenant, Grant.base_id == KnowledgeBase.id)).where(
KnowledgeBase.tenant == user["tenant_id"], Grant.user_id == user["user_id"])))
if hasattr(self,"organization"):
extra=self.organization.allowed_bases(user)
rows=list({b.id:b for b in rows+list(db.scalars(select(KnowledgeBase).where(KnowledgeBase.tenant==user["tenant_id"],KnowledgeBase.id.in_(extra))))}.values())
return [b for b in rows if (db.get(LibraryConfig, (b.tenant, b.id)) is None or
db.get(LibraryConfig, (b.tenant, b.id)).status == "enabled")]
def bases(self, user):
return self.library_list(user)
def _scope(self, db, user, wid, requested=None):
allowed = {b.id: b for b in self._bases(db, user)}
ids = list(dict.fromkeys(ALIASES.get(b, b) for b in (requested or allowed)))
if any(b not in allowed for b in ids):
raise missing()
shared = [bid for bid in ids if allowed[bid].visibility == "shared"]
personal = [bid for bid in ids if allowed[bid].visibility == "personal"]
return and_(Document.tenant == user["tenant_id"], or_(
and_(Document.base_id.in_(shared), Document.owner.is_(None), Document.workspace.is_(None)),
and_(Document.base_id.in_(personal), Document.owner == user["user_id"], Document.workspace == wid)))
@staticmethod
def _valid():
today = datetime.now(timezone.utc).date().isoformat()
return and_(Document.status == "ready",
or_(Document.valid_from.is_(None), Document.valid_from <= today),
or_(Document.valid_to.is_(None), Document.valid_to >= today))
def _citation(self, chunk, doc, score=None, full=False, sources=None):
result = {"citation_id": chunk.id, "chunk_id": chunk.id, "document_id": doc.document_id,
"document_version": int(doc.version) if doc.version.isdigit() else doc.version,
"base_id": doc.base_id, "title": doc.title, "locator": chunk.locator,
"snippet": chunk.text[:800], "content_hash": doc.sha256,
"source_type": doc.source_type, "provider_mode": "persistent"}
if sources is not None:
if sources.get(doc.id):result["provenance"]=sources[doc.id]
else:
with self.Session() as db:
source=db.get(DocumentSource,doc.id)
if source:result["provenance"]=source.body
if score is not None:
result["score"] = round(score, 6)
if full:
result["text"] = chunk.text
return result
@traced("Knowledge Service")
def search(self, user, wid, query, base_ids, top_k):
self.workspace.assert_access(user, wid)
if not isinstance(query, str) or not query.strip() or len(query) > 2000 or not 1 <= top_k <= 10:
raise KnowledgeError(422, "INVALID_SEARCH", "检索问题和条数不符合约束。")
with self.Session() as db:
rows = db.execute(select(Chunk, Document).join(Document).outerjoin(DocumentInfo,DocumentInfo.document_key==Document.id).where(
self._scope(db, user, wid, base_ids), self._valid(),or_(DocumentInfo.document_key.is_(None),DocumentInfo.publication=='published'))).all()
# Use the most recently imported valid immutable version in this authorized scope.
latest = {}
for _, doc in rows:
marker = (doc.created_at, doc.id)
if marker > latest.get(doc.document_id, ("", "")):
latest[doc.document_id] = marker
rows = [(c, d) for c, d in rows if (d.created_at, d.id) == latest[d.document_id]]
sources={s.document_key:s.body for s in db.scalars(select(DocumentSource).where(DocumentSource.document_key.in_({d.id for c,d in rows})))}
with span('Knowledge Recall','authorized current chunks',{'base_ids':base_ids,'query':query}) as stage:
stage['value']={'authorized_chunk_count':len(rows),'authorized_document_count':len({d.id for c,d in rows}),'filter':'tenant + user/group grant + personal owner/workspace + published + effective dates + latest version'}
ranked=rank(query,rows,max(1,len(rows)))
picked={c.id for score,c,d in ranked[:top_k]}
scores={c.id:score for score,c,d in ranked}
# Persist a full authorized chunk ledger; details are paginated independently of span caps.
from observability import chunk_ledger
ledger=[{**self._citation(c,d,scores.get(c.id,0),sources=sources), 'ordinal':c.ordinal,'decision':'selected' if c.id in picked else 'below_top_k' if c.id in scores else 'no_term_match'} for c,d in rows]
chunk_ledger(ledger)
with span('Knowledge Recall','select ranked chunks',{'algorithm':'BM25','top_k':top_k}) as stage:
stage['value']={'candidate_count':len(rows),'positive_score_count':len(ranked),'selected':[self._citation(c,d,score,sources=sources) for score,c,d in ranked[:top_k]],'chunk_ledger_count':len(ledger)}
return {"items": [self._citation(c, d, score,sources=sources) for score, c, d in ranked[:top_k]],
"next_cursor": None, "mode": self.mode}
@traced("Knowledge Service")
def citation(self, user, cid, workspace_id=None):
if workspace_id:
self.workspace.assert_access(user, workspace_id)
with self.Session() as db:
# A private citation needs both the owner and current workspace. IDs are not capabilities.
row = db.execute(select(Chunk, Document).join(Document).where(
Chunk.id == cid, self._scope(db, user, workspace_id), self._valid())).first()
if row is None or not self._published(db, row[1]):
raise missing()
return self._citation(*row, full=True)
def document_source(self, user, cid, workspace_id=None):
citation = self.citation(user, cid, workspace_id)
filename, content = self.original(user, cid, workspace_id)
try:
text = '\n\n'.join(text for _, text in extract_units(filename, content))
except ParseError:
raise KnowledgeError(422, 'ORIGINAL_PREVIEW_UNAVAILABLE', '原文件暂不支持文字预览,请下载查看。') from None
return {**citation, 'filename': filename, 'text': text, 'locator': '完整原文件', 'preview_scope': 'document'}
def documents(self, user, wid):
self.workspace.assert_access(user, wid)
with self.Session() as db:
rows = db.scalars(select(Document).where(self._scope(db, user, wid)).order_by(Document.id)).all()
return {"items": [{"document_id": d.document_id, "document_version": d.version,
"title": d.title, "base_id": d.base_id, "status": d.status,
"error_code": d.error_code, "source_type": d.source_type,
"content_hash": d.sha256} for d in rows], "next_cursor": None}
@traced('Knowledge Service','read current meeting attachments')
def meeting_materials(self,user,wid):
self.workspace.assert_access(user,wid)
with self.Session() as db:
rows=db.execute(select(Chunk,Document).join(Document).where(self._scope(db,user,wid,['kb-personal']),self._valid()).order_by(Document.created_at,Chunk.ordinal)).all()
rows=[(c,d) for c,d in rows if self._published(db,d)]
latest={}
for _,d in rows:latest[d.document_id]=max(latest.get(d.document_id,('', '')),(d.created_at,d.id))
rows=[(c,d) for c,d in rows if latest[d.document_id]==(d.created_at,d.id)]
if sum(len(c.text) for c,_ in rows)>32000:
raise KnowledgeError(413,'MEETING_MATERIAL_TOO_LONG','当前事项转写超过32000字,请按议题拆分到不同事项后整理。')
return {'items':[self._citation(c,d) for c,d in rows]}
@traced("Knowledge Service")
def _save(self, *, tenant, document_id, version, base_id, title, content, filename,
owner=None, workspace=None, source_type="attachment", valid_from=None, valid_to=None, provenance=None):
"""Atomic immutable version+chunks. Same identity with changed bytes is a conflict."""
sha = hashlib.sha256(content).hexdigest()
key = "doc-" + digest(json.dumps([tenant, document_id, version]))
try:
chunks, error = extract(filename, content), None
except ParseError as exc:
chunks, error = [], str(exc)
with self.lock, self.Session.begin() as db:
base = db.get(KnowledgeBase, (tenant, base_id))
if base is None or (base.visibility == "personal" and (not owner or not workspace)):
raise missing()
if base.visibility == "shared" and (owner is not None or workspace is not None):
raise KnowledgeError(422, "INVALID_DOCUMENT_SCOPE", "共享库不能存放私人事项材料。")
old = db.get(Document, key)
previous = db.scalar(select(Document).where(Document.tenant == tenant, Document.document_id == document_id))
if previous and (previous.base_id, previous.owner, previous.workspace) != (base_id, owner, workspace):
raise KnowledgeError(409, "DOCUMENT_SCOPE_CONFLICT", "文档新版本不能改变授权归属。")
if old is not None:
if (old.sha256, old.base_id, old.owner, old.workspace) != (sha, base_id, owner, workspace):
raise KnowledgeError(409, "DOCUMENT_VERSION_CONFLICT", "同一文档版本不可替换内容或归属。")
self._info(db, old, filename, content, chunks)
if provenance is not None:
source=db.get(DocumentSource,key)
if source:source.body={**provenance,"content_sha256":sha,"sha256":sha}
else:db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha}))
return old
doc = Document(id=key, tenant=tenant, document_id=document_id, version=version,
base_id=base_id, title=title, owner=owner, workspace=workspace, source_type=source_type,
sha256=sha, status="failed" if error else "ready", error_code=error,
valid_from=valid_from, valid_to=valid_to,
created_at=datetime.now(timezone.utc).isoformat())
db.add(doc)
db.flush()
self._info(db, doc, filename, content, chunks)
if provenance is not None:
db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha}))
for ordinal, (locator, text, terms) in enumerate(chunks, 1):
db.add(Chunk(id="cite-" + digest(key + ":" + str(ordinal)), document_key=key,
ordinal=ordinal, locator=locator, text=text, terms=terms))
return doc
@traced("Knowledge Service")
def consume_event(self, envelope):
if envelope["event_type"] != "document.ingest.requested":
return False
user = {"tenant_id": envelope["tenant_id"], "user_id": envelope["owner_user_id"]}
aid, wid = envelope["payload"]["attachment_id"], envelope["workspace_id"]
info = self.workspace.attachment_info(user, aid)
if info["workspace_id"] != wid:
raise missing()
with self.Session() as db:
if "kb-personal" not in {b.id for b in self._bases(db, user)}:
raise missing()
doc = self._save(tenant=user["tenant_id"], document_id=aid, version="1", base_id="kb-personal",
title=info["filename"], filename=info["filename"], content=self.workspace.attachment_content(user, aid),
owner=user["user_id"], workspace=wid)
# DB/storage exceptions propagate so the durable outbox remains pending for retry.
self.workspace.apply_ingest_result(user, aid, doc.status, doc.document_id, doc.error_code)
return True
def import_seeds(self, private_mapping=None):
"""Trusted local operator entry: private_mapping maps demo key to real principal+workspace."""
manifest = json.loads((SEEDS / "manifest.json").read_text(encoding="utf-8-sig"))
imported = []
for meta in manifest["documents"]:
owner, wid = None, None
if meta["owner_user_id"]:
mapping = (private_mapping or {}).get(meta["workspace_key"])
if mapping is None:
continue
user, wid = mapping["principal"], mapping["workspace_id"]
self.workspace.assert_access(user, wid)
if user["tenant_id"] != "demo-tenant":
raise missing()
owner = user["user_id"]
content = (SEEDS / meta["file"]).read_bytes().replace(b"\r\n", b"\n")
if hashlib.sha256(content).hexdigest() != meta["sha256"]:
raise KnowledgeError(422, "SEED_HASH_MISMATCH", "种子资料校验失败。")
doc = self._save(tenant="demo-tenant", document_id=meta["document_id"], version=meta["version"],
base_id=ALIASES[meta["kb_id"]], title=meta["title"], content=content, filename=meta["file"],
owner=owner, workspace=wid, source_type=meta["source_type"],
valid_from=meta["valid_from"], valid_to=meta["valid_to"])
imported.append({"document_id": doc.document_id, "status": doc.status})
return imported