File size: 17,046 Bytes
99caad5
 
 
 
 
 
 
 
 
 
 
67e287e
99caad5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67e287e
 
 
 
 
 
 
 
 
 
99caad5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
"""Single-process SQL Knowledge Port, sharing an engine but owning kb_* tables."""
from __future__ import annotations
from observability import traced, span
import hashlib
import json
import os
import threading
from datetime import datetime, timezone
from pathlib import Path
from sqlalchemy import and_, or_, select
from .models import Base, KnowledgeBase, Grant, Document, Chunk, KnowledgeError
from .parsing import ParseError, extract, extract_units, rank
from .management import Management
from .models import LibraryConfig,DocumentInfo,DocumentSource

BASES = [
    ("kb-rules", "制度与工作依据库", "shared"),
    ("kb-history", "历史公文案例库", "shared"),
    ("kb-templates", "公文模板与要素库", "shared"),
    ("kb-circulation", "发文与办理流程库", "shared"),
    ("kb-personal", "个人来文与事项材料库", "personal"),
    ("kb-office", "邮件与会议知识库", "shared"),
]
ALIASES = {**{f"KB0{i+1}": row[0] for i, row in enumerate(BASES)}, "kb-policy": "kb-rules"}
SEEDS = Path(__file__).resolve().parents[1] / "docs" / "05_知识库种子素材"


def digest(value):
    return hashlib.sha256(value.encode("utf-8")).hexdigest()


def missing():
    return KnowledgeError(404, "RESOURCE_NOT_FOUND", "资源不存在或无权访问。")


class KnowledgeService(Management):
    mode = "sql-bm25-persistent"

    def __init__(self, workspace, seed_demo=None):
        self.workspace = workspace
        self.Session = workspace.Session
        self.lock = threading.RLock()
        if os.getenv("WORKOS_WORKSPACE_AUTO_SCHEMA", "1") == "1":
            Base.metadata.create_all(workspace.engine)
        should_seed = seed_demo if seed_demo is not None else os.getenv("WORKOS_KNOWLEDGE_SEED_DEMO", "1") == "1"
        if should_seed:
            self.seed_demo_catalog()

    def seed_demo_catalog(self):
        """Grant demo users explicitly; never infer grants for other tenants/users."""
        from .seed import corpus_plan, import_catalog
        plan=corpus_plan()  # No partial import for a corrupt bundled file.
        with self.lock, self.Session.begin() as db:
            for bid, name, visibility in BASES:
                if db.get(KnowledgeBase, ("demo-tenant", bid)) is None:
                    db.add(KnowledgeBase(tenant="demo-tenant", id=bid, name=name, visibility=visibility))
                    for uid in ("u-lin", "u-zhou", "u-admin"):
                        db.add(Grant(tenant="demo-tenant", base_id=bid, user_id=uid))
                if db.get(LibraryConfig,('demo-tenant',bid)) is None:
                    category={'kb-rules':'制度依据','kb-history':'写作案例','kb-templates':'公文模板','kb-circulation':'办理流程','kb-personal':'事项附件','kb-office':'邮件与会议'}[bid]
                    db.add(LibraryConfig(tenant='demo-tenant',base_id=bid,description=name+':可核对出处的办公合成样例。' if visibility=='shared' else '本人当前事项上传的来文与会议转写。',category=category,status='enabled',revision=1))
        import_catalog(self, plan)

    def _bases(self, db, user):
        rows = list(db.scalars(select(KnowledgeBase).join(Grant, and_(
            Grant.tenant == KnowledgeBase.tenant, Grant.base_id == KnowledgeBase.id)).where(
            KnowledgeBase.tenant == user["tenant_id"], Grant.user_id == user["user_id"])))
        if hasattr(self,"organization"):
            extra=self.organization.allowed_bases(user)
            rows=list({b.id:b for b in rows+list(db.scalars(select(KnowledgeBase).where(KnowledgeBase.tenant==user["tenant_id"],KnowledgeBase.id.in_(extra))))}.values())
        return [b for b in rows if (db.get(LibraryConfig, (b.tenant, b.id)) is None or
                                   db.get(LibraryConfig, (b.tenant, b.id)).status == "enabled")]

    def bases(self, user):
        return self.library_list(user)

    def _scope(self, db, user, wid, requested=None):
        allowed = {b.id: b for b in self._bases(db, user)}
        ids = list(dict.fromkeys(ALIASES.get(b, b) for b in (requested or allowed)))
        if any(b not in allowed for b in ids):
            raise missing()
        shared = [bid for bid in ids if allowed[bid].visibility == "shared"]
        personal = [bid for bid in ids if allowed[bid].visibility == "personal"]
        return and_(Document.tenant == user["tenant_id"], or_(
            and_(Document.base_id.in_(shared), Document.owner.is_(None), Document.workspace.is_(None)),
            and_(Document.base_id.in_(personal), Document.owner == user["user_id"], Document.workspace == wid)))

    @staticmethod
    def _valid():
        today = datetime.now(timezone.utc).date().isoformat()
        return and_(Document.status == "ready",
            or_(Document.valid_from.is_(None), Document.valid_from <= today),
            or_(Document.valid_to.is_(None), Document.valid_to >= today))

    def _citation(self, chunk, doc, score=None, full=False, sources=None):
        result = {"citation_id": chunk.id, "chunk_id": chunk.id, "document_id": doc.document_id,
            "document_version": int(doc.version) if doc.version.isdigit() else doc.version,
            "base_id": doc.base_id, "title": doc.title, "locator": chunk.locator,
            "snippet": chunk.text[:800], "content_hash": doc.sha256,
            "source_type": doc.source_type, "provider_mode": "persistent"}
        if sources is not None:
            if sources.get(doc.id):result["provenance"]=sources[doc.id]
        else:
            with self.Session() as db:
                source=db.get(DocumentSource,doc.id)
                if source:result["provenance"]=source.body
        if score is not None:
            result["score"] = round(score, 6)
        if full:
            result["text"] = chunk.text
        return result

    @traced("Knowledge Service")
    def search(self, user, wid, query, base_ids, top_k):
        self.workspace.assert_access(user, wid)
        if not isinstance(query, str) or not query.strip() or len(query) > 2000 or not 1 <= top_k <= 10:
            raise KnowledgeError(422, "INVALID_SEARCH", "检索问题和条数不符合约束。")
        with self.Session() as db:
            rows = db.execute(select(Chunk, Document).join(Document).outerjoin(DocumentInfo,DocumentInfo.document_key==Document.id).where(
                self._scope(db, user, wid, base_ids), self._valid(),or_(DocumentInfo.document_key.is_(None),DocumentInfo.publication=='published'))).all()
            # Use the most recently imported valid immutable version in this authorized scope.
            latest = {}
            for _, doc in rows:
                marker = (doc.created_at, doc.id)
                if marker > latest.get(doc.document_id, ("", "")):
                    latest[doc.document_id] = marker
            rows = [(c, d) for c, d in rows if (d.created_at, d.id) == latest[d.document_id]]
            sources={s.document_key:s.body for s in db.scalars(select(DocumentSource).where(DocumentSource.document_key.in_({d.id for c,d in rows})))}
            with span('Knowledge Recall','authorized current chunks',{'base_ids':base_ids,'query':query}) as stage:
                stage['value']={'authorized_chunk_count':len(rows),'authorized_document_count':len({d.id for c,d in rows}),'filter':'tenant + user/group grant + personal owner/workspace + published + effective dates + latest version'}
            ranked=rank(query,rows,max(1,len(rows)))
            picked={c.id for score,c,d in ranked[:top_k]}
            scores={c.id:score for score,c,d in ranked}
            # Persist a full authorized chunk ledger; details are paginated independently of span caps.
            from observability import chunk_ledger
            ledger=[{**self._citation(c,d,scores.get(c.id,0),sources=sources), 'ordinal':c.ordinal,'decision':'selected' if c.id in picked else 'below_top_k' if c.id in scores else 'no_term_match'} for c,d in rows]
            chunk_ledger(ledger)
            with span('Knowledge Recall','select ranked chunks',{'algorithm':'BM25','top_k':top_k}) as stage:
                stage['value']={'candidate_count':len(rows),'positive_score_count':len(ranked),'selected':[self._citation(c,d,score,sources=sources) for score,c,d in ranked[:top_k]],'chunk_ledger_count':len(ledger)}
            return {"items": [self._citation(c, d, score,sources=sources) for score, c, d in ranked[:top_k]],
                "next_cursor": None, "mode": self.mode}

    @traced("Knowledge Service")
    def citation(self, user, cid, workspace_id=None):
        if workspace_id:
            self.workspace.assert_access(user, workspace_id)
        with self.Session() as db:
            # A private citation needs both the owner and current workspace. IDs are not capabilities.
            row = db.execute(select(Chunk, Document).join(Document).where(
                Chunk.id == cid, self._scope(db, user, workspace_id), self._valid())).first()
            if row is None or not self._published(db, row[1]):
                raise missing()
            return self._citation(*row, full=True)

    def document_source(self, user, cid, workspace_id=None):
        citation = self.citation(user, cid, workspace_id)
        filename, content = self.original(user, cid, workspace_id)
        try:
            text = '\n\n'.join(text for _, text in extract_units(filename, content))
        except ParseError:
            raise KnowledgeError(422, 'ORIGINAL_PREVIEW_UNAVAILABLE', '原文件暂不支持文字预览,请下载查看。') from None
        return {**citation, 'filename': filename, 'text': text, 'locator': '完整原文件', 'preview_scope': 'document'}

    def documents(self, user, wid):
        self.workspace.assert_access(user, wid)
        with self.Session() as db:
            rows = db.scalars(select(Document).where(self._scope(db, user, wid)).order_by(Document.id)).all()
            return {"items": [{"document_id": d.document_id, "document_version": d.version,
                "title": d.title, "base_id": d.base_id, "status": d.status,
                "error_code": d.error_code, "source_type": d.source_type,
                "content_hash": d.sha256} for d in rows], "next_cursor": None}

    @traced('Knowledge Service','read current meeting attachments')
    def meeting_materials(self,user,wid):
        self.workspace.assert_access(user,wid)
        with self.Session() as db:
            rows=db.execute(select(Chunk,Document).join(Document).where(self._scope(db,user,wid,['kb-personal']),self._valid()).order_by(Document.created_at,Chunk.ordinal)).all()
            rows=[(c,d) for c,d in rows if self._published(db,d)]
            latest={}
            for _,d in rows:latest[d.document_id]=max(latest.get(d.document_id,('', '')),(d.created_at,d.id))
            rows=[(c,d) for c,d in rows if latest[d.document_id]==(d.created_at,d.id)]
            if sum(len(c.text) for c,_ in rows)>32000:
                raise KnowledgeError(413,'MEETING_MATERIAL_TOO_LONG','当前事项转写超过32000字,请按议题拆分到不同事项后整理。')
            return {'items':[self._citation(c,d) for c,d in rows]}

    @traced("Knowledge Service")
    def _save(self, *, tenant, document_id, version, base_id, title, content, filename,

              owner=None, workspace=None, source_type="attachment", valid_from=None, valid_to=None, provenance=None):
        """Atomic immutable version+chunks. Same identity with changed bytes is a conflict."""
        sha = hashlib.sha256(content).hexdigest()
        key = "doc-" + digest(json.dumps([tenant, document_id, version]))
        try:
            chunks, error = extract(filename, content), None
        except ParseError as exc:
            chunks, error = [], str(exc)
        with self.lock, self.Session.begin() as db:
            base = db.get(KnowledgeBase, (tenant, base_id))
            if base is None or (base.visibility == "personal" and (not owner or not workspace)):
                raise missing()
            if base.visibility == "shared" and (owner is not None or workspace is not None):
                raise KnowledgeError(422, "INVALID_DOCUMENT_SCOPE", "共享库不能存放私人事项材料。")
            old = db.get(Document, key)
            previous = db.scalar(select(Document).where(Document.tenant == tenant, Document.document_id == document_id))
            if previous and (previous.base_id, previous.owner, previous.workspace) != (base_id, owner, workspace):
                raise KnowledgeError(409, "DOCUMENT_SCOPE_CONFLICT", "文档新版本不能改变授权归属。")
            if old is not None:
                if (old.sha256, old.base_id, old.owner, old.workspace) != (sha, base_id, owner, workspace):
                    raise KnowledgeError(409, "DOCUMENT_VERSION_CONFLICT", "同一文档版本不可替换内容或归属。")
                self._info(db, old, filename, content, chunks)
                if provenance is not None:
                    source=db.get(DocumentSource,key)
                    if source:source.body={**provenance,"content_sha256":sha,"sha256":sha}
                    else:db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha}))
                return old
            doc = Document(id=key, tenant=tenant, document_id=document_id, version=version,
                base_id=base_id, title=title, owner=owner, workspace=workspace, source_type=source_type,
                sha256=sha, status="failed" if error else "ready", error_code=error,
                valid_from=valid_from, valid_to=valid_to,
                created_at=datetime.now(timezone.utc).isoformat())
            db.add(doc)
            db.flush()
            self._info(db, doc, filename, content, chunks)
            if provenance is not None:
                db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha}))
            for ordinal, (locator, text, terms) in enumerate(chunks, 1):
                db.add(Chunk(id="cite-" + digest(key + ":" + str(ordinal)), document_key=key,
                    ordinal=ordinal, locator=locator, text=text, terms=terms))
            return doc

    @traced("Knowledge Service")
    def consume_event(self, envelope):
        if envelope["event_type"] != "document.ingest.requested":
            return False
        user = {"tenant_id": envelope["tenant_id"], "user_id": envelope["owner_user_id"]}
        aid, wid = envelope["payload"]["attachment_id"], envelope["workspace_id"]
        info = self.workspace.attachment_info(user, aid)
        if info["workspace_id"] != wid:
            raise missing()
        with self.Session() as db:
            if "kb-personal" not in {b.id for b in self._bases(db, user)}:
                raise missing()
        doc = self._save(tenant=user["tenant_id"], document_id=aid, version="1", base_id="kb-personal",
            title=info["filename"], filename=info["filename"], content=self.workspace.attachment_content(user, aid),
            owner=user["user_id"], workspace=wid)
        # DB/storage exceptions propagate so the durable outbox remains pending for retry.
        self.workspace.apply_ingest_result(user, aid, doc.status, doc.document_id, doc.error_code)
        return True

    def import_seeds(self, private_mapping=None):
        """Trusted local operator entry: private_mapping maps demo key to real principal+workspace."""
        manifest = json.loads((SEEDS / "manifest.json").read_text(encoding="utf-8-sig"))
        imported = []
        for meta in manifest["documents"]:
            owner, wid = None, None
            if meta["owner_user_id"]:
                mapping = (private_mapping or {}).get(meta["workspace_key"])
                if mapping is None:
                    continue
                user, wid = mapping["principal"], mapping["workspace_id"]
                self.workspace.assert_access(user, wid)
                if user["tenant_id"] != "demo-tenant":
                    raise missing()
                owner = user["user_id"]
            content = (SEEDS / meta["file"]).read_bytes().replace(b"\r\n", b"\n")
            if hashlib.sha256(content).hexdigest() != meta["sha256"]:
                raise KnowledgeError(422, "SEED_HASH_MISMATCH", "种子资料校验失败。")
            doc = self._save(tenant="demo-tenant", document_id=meta["document_id"], version=meta["version"],
                base_id=ALIASES[meta["kb_id"]], title=meta["title"], content=content, filename=meta["file"],
                owner=owner, workspace=wid, source_type=meta["source_type"],
                valid_from=meta["valid_from"], valid_to=meta["valid_to"])
            imported.append({"document_id": doc.document_id, "status": doc.status})
        return imported