Spaces:
Sleeping
Sleeping
Download KnowledgeService/service.py from dellweb3/workos-office: direct link, hf CLI and curl.
- Browser
- Download file 17 kB
-
https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/service.py
- Command line
-
hf download hf://spaces/dellweb3/workos-office/KnowledgeService/service.py
-
curl -L -o service.py https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/service.py
17 kB
| """Single-process SQL Knowledge Port, sharing an engine but owning kb_* tables.""" | |
| from __future__ import annotations | |
| from observability import traced, span | |
| import hashlib | |
| import json | |
| import os | |
| import threading | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from sqlalchemy import and_, or_, select | |
| from .models import Base, KnowledgeBase, Grant, Document, Chunk, KnowledgeError | |
| from .parsing import ParseError, extract, extract_units, rank | |
| from .management import Management | |
| from .models import LibraryConfig,DocumentInfo,DocumentSource | |
| BASES = [ | |
| ("kb-rules", "制度与工作依据库", "shared"), | |
| ("kb-history", "历史公文案例库", "shared"), | |
| ("kb-templates", "公文模板与要素库", "shared"), | |
| ("kb-circulation", "发文与办理流程库", "shared"), | |
| ("kb-personal", "个人来文与事项材料库", "personal"), | |
| ("kb-office", "邮件与会议知识库", "shared"), | |
| ] | |
| ALIASES = {**{f"KB0{i+1}": row[0] for i, row in enumerate(BASES)}, "kb-policy": "kb-rules"} | |
| SEEDS = Path(__file__).resolve().parents[1] / "docs" / "05_知识库种子素材" | |
| def digest(value): | |
| return hashlib.sha256(value.encode("utf-8")).hexdigest() | |
| def missing(): | |
| return KnowledgeError(404, "RESOURCE_NOT_FOUND", "资源不存在或无权访问。") | |
| class KnowledgeService(Management): | |
| mode = "sql-bm25-persistent" | |
| def __init__(self, workspace, seed_demo=None): | |
| self.workspace = workspace | |
| self.Session = workspace.Session | |
| self.lock = threading.RLock() | |
| if os.getenv("WORKOS_WORKSPACE_AUTO_SCHEMA", "1") == "1": | |
| Base.metadata.create_all(workspace.engine) | |
| should_seed = seed_demo if seed_demo is not None else os.getenv("WORKOS_KNOWLEDGE_SEED_DEMO", "1") == "1" | |
| if should_seed: | |
| self.seed_demo_catalog() | |
| def seed_demo_catalog(self): | |
| """Grant demo users explicitly; never infer grants for other tenants/users.""" | |
| from .seed import corpus_plan, import_catalog | |
| plan=corpus_plan() # No partial import for a corrupt bundled file. | |
| with self.lock, self.Session.begin() as db: | |
| for bid, name, visibility in BASES: | |
| if db.get(KnowledgeBase, ("demo-tenant", bid)) is None: | |
| db.add(KnowledgeBase(tenant="demo-tenant", id=bid, name=name, visibility=visibility)) | |
| for uid in ("u-lin", "u-zhou", "u-admin"): | |
| db.add(Grant(tenant="demo-tenant", base_id=bid, user_id=uid)) | |
| if db.get(LibraryConfig,('demo-tenant',bid)) is None: | |
| category={'kb-rules':'制度依据','kb-history':'写作案例','kb-templates':'公文模板','kb-circulation':'办理流程','kb-personal':'事项附件','kb-office':'邮件与会议'}[bid] | |
| db.add(LibraryConfig(tenant='demo-tenant',base_id=bid,description=name+':可核对出处的办公合成样例。' if visibility=='shared' else '本人当前事项上传的来文与会议转写。',category=category,status='enabled',revision=1)) | |
| import_catalog(self, plan) | |
| def _bases(self, db, user): | |
| rows = list(db.scalars(select(KnowledgeBase).join(Grant, and_( | |
| Grant.tenant == KnowledgeBase.tenant, Grant.base_id == KnowledgeBase.id)).where( | |
| KnowledgeBase.tenant == user["tenant_id"], Grant.user_id == user["user_id"]))) | |
| if hasattr(self,"organization"): | |
| extra=self.organization.allowed_bases(user) | |
| rows=list({b.id:b for b in rows+list(db.scalars(select(KnowledgeBase).where(KnowledgeBase.tenant==user["tenant_id"],KnowledgeBase.id.in_(extra))))}.values()) | |
| return [b for b in rows if (db.get(LibraryConfig, (b.tenant, b.id)) is None or | |
| db.get(LibraryConfig, (b.tenant, b.id)).status == "enabled")] | |
| def bases(self, user): | |
| return self.library_list(user) | |
| def _scope(self, db, user, wid, requested=None): | |
| allowed = {b.id: b for b in self._bases(db, user)} | |
| ids = list(dict.fromkeys(ALIASES.get(b, b) for b in (requested or allowed))) | |
| if any(b not in allowed for b in ids): | |
| raise missing() | |
| shared = [bid for bid in ids if allowed[bid].visibility == "shared"] | |
| personal = [bid for bid in ids if allowed[bid].visibility == "personal"] | |
| return and_(Document.tenant == user["tenant_id"], or_( | |
| and_(Document.base_id.in_(shared), Document.owner.is_(None), Document.workspace.is_(None)), | |
| and_(Document.base_id.in_(personal), Document.owner == user["user_id"], Document.workspace == wid))) | |
| def _valid(): | |
| today = datetime.now(timezone.utc).date().isoformat() | |
| return and_(Document.status == "ready", | |
| or_(Document.valid_from.is_(None), Document.valid_from <= today), | |
| or_(Document.valid_to.is_(None), Document.valid_to >= today)) | |
| def _citation(self, chunk, doc, score=None, full=False, sources=None): | |
| result = {"citation_id": chunk.id, "chunk_id": chunk.id, "document_id": doc.document_id, | |
| "document_version": int(doc.version) if doc.version.isdigit() else doc.version, | |
| "base_id": doc.base_id, "title": doc.title, "locator": chunk.locator, | |
| "snippet": chunk.text[:800], "content_hash": doc.sha256, | |
| "source_type": doc.source_type, "provider_mode": "persistent"} | |
| if sources is not None: | |
| if sources.get(doc.id):result["provenance"]=sources[doc.id] | |
| else: | |
| with self.Session() as db: | |
| source=db.get(DocumentSource,doc.id) | |
| if source:result["provenance"]=source.body | |
| if score is not None: | |
| result["score"] = round(score, 6) | |
| if full: | |
| result["text"] = chunk.text | |
| return result | |
| def search(self, user, wid, query, base_ids, top_k): | |
| self.workspace.assert_access(user, wid) | |
| if not isinstance(query, str) or not query.strip() or len(query) > 2000 or not 1 <= top_k <= 10: | |
| raise KnowledgeError(422, "INVALID_SEARCH", "检索问题和条数不符合约束。") | |
| with self.Session() as db: | |
| rows = db.execute(select(Chunk, Document).join(Document).outerjoin(DocumentInfo,DocumentInfo.document_key==Document.id).where( | |
| self._scope(db, user, wid, base_ids), self._valid(),or_(DocumentInfo.document_key.is_(None),DocumentInfo.publication=='published'))).all() | |
| # Use the most recently imported valid immutable version in this authorized scope. | |
| latest = {} | |
| for _, doc in rows: | |
| marker = (doc.created_at, doc.id) | |
| if marker > latest.get(doc.document_id, ("", "")): | |
| latest[doc.document_id] = marker | |
| rows = [(c, d) for c, d in rows if (d.created_at, d.id) == latest[d.document_id]] | |
| sources={s.document_key:s.body for s in db.scalars(select(DocumentSource).where(DocumentSource.document_key.in_({d.id for c,d in rows})))} | |
| with span('Knowledge Recall','authorized current chunks',{'base_ids':base_ids,'query':query}) as stage: | |
| stage['value']={'authorized_chunk_count':len(rows),'authorized_document_count':len({d.id for c,d in rows}),'filter':'tenant + user/group grant + personal owner/workspace + published + effective dates + latest version'} | |
| ranked=rank(query,rows,max(1,len(rows))) | |
| picked={c.id for score,c,d in ranked[:top_k]} | |
| scores={c.id:score for score,c,d in ranked} | |
| # Persist a full authorized chunk ledger; details are paginated independently of span caps. | |
| from observability import chunk_ledger | |
| ledger=[{**self._citation(c,d,scores.get(c.id,0),sources=sources), 'ordinal':c.ordinal,'decision':'selected' if c.id in picked else 'below_top_k' if c.id in scores else 'no_term_match'} for c,d in rows] | |
| chunk_ledger(ledger) | |
| with span('Knowledge Recall','select ranked chunks',{'algorithm':'BM25','top_k':top_k}) as stage: | |
| stage['value']={'candidate_count':len(rows),'positive_score_count':len(ranked),'selected':[self._citation(c,d,score,sources=sources) for score,c,d in ranked[:top_k]],'chunk_ledger_count':len(ledger)} | |
| return {"items": [self._citation(c, d, score,sources=sources) for score, c, d in ranked[:top_k]], | |
| "next_cursor": None, "mode": self.mode} | |
| def citation(self, user, cid, workspace_id=None): | |
| if workspace_id: | |
| self.workspace.assert_access(user, workspace_id) | |
| with self.Session() as db: | |
| # A private citation needs both the owner and current workspace. IDs are not capabilities. | |
| row = db.execute(select(Chunk, Document).join(Document).where( | |
| Chunk.id == cid, self._scope(db, user, workspace_id), self._valid())).first() | |
| if row is None or not self._published(db, row[1]): | |
| raise missing() | |
| return self._citation(*row, full=True) | |
| def document_source(self, user, cid, workspace_id=None): | |
| citation = self.citation(user, cid, workspace_id) | |
| filename, content = self.original(user, cid, workspace_id) | |
| try: | |
| text = '\n\n'.join(text for _, text in extract_units(filename, content)) | |
| except ParseError: | |
| raise KnowledgeError(422, 'ORIGINAL_PREVIEW_UNAVAILABLE', '原文件暂不支持文字预览,请下载查看。') from None | |
| return {**citation, 'filename': filename, 'text': text, 'locator': '完整原文件', 'preview_scope': 'document'} | |
| def documents(self, user, wid): | |
| self.workspace.assert_access(user, wid) | |
| with self.Session() as db: | |
| rows = db.scalars(select(Document).where(self._scope(db, user, wid)).order_by(Document.id)).all() | |
| return {"items": [{"document_id": d.document_id, "document_version": d.version, | |
| "title": d.title, "base_id": d.base_id, "status": d.status, | |
| "error_code": d.error_code, "source_type": d.source_type, | |
| "content_hash": d.sha256} for d in rows], "next_cursor": None} | |
| def meeting_materials(self,user,wid): | |
| self.workspace.assert_access(user,wid) | |
| with self.Session() as db: | |
| rows=db.execute(select(Chunk,Document).join(Document).where(self._scope(db,user,wid,['kb-personal']),self._valid()).order_by(Document.created_at,Chunk.ordinal)).all() | |
| rows=[(c,d) for c,d in rows if self._published(db,d)] | |
| latest={} | |
| for _,d in rows:latest[d.document_id]=max(latest.get(d.document_id,('', '')),(d.created_at,d.id)) | |
| rows=[(c,d) for c,d in rows if latest[d.document_id]==(d.created_at,d.id)] | |
| if sum(len(c.text) for c,_ in rows)>32000: | |
| raise KnowledgeError(413,'MEETING_MATERIAL_TOO_LONG','当前事项转写超过32000字,请按议题拆分到不同事项后整理。') | |
| return {'items':[self._citation(c,d) for c,d in rows]} | |
| def _save(self, *, tenant, document_id, version, base_id, title, content, filename, | |
| owner=None, workspace=None, source_type="attachment", valid_from=None, valid_to=None, provenance=None): | |
| """Atomic immutable version+chunks. Same identity with changed bytes is a conflict.""" | |
| sha = hashlib.sha256(content).hexdigest() | |
| key = "doc-" + digest(json.dumps([tenant, document_id, version])) | |
| try: | |
| chunks, error = extract(filename, content), None | |
| except ParseError as exc: | |
| chunks, error = [], str(exc) | |
| with self.lock, self.Session.begin() as db: | |
| base = db.get(KnowledgeBase, (tenant, base_id)) | |
| if base is None or (base.visibility == "personal" and (not owner or not workspace)): | |
| raise missing() | |
| if base.visibility == "shared" and (owner is not None or workspace is not None): | |
| raise KnowledgeError(422, "INVALID_DOCUMENT_SCOPE", "共享库不能存放私人事项材料。") | |
| old = db.get(Document, key) | |
| previous = db.scalar(select(Document).where(Document.tenant == tenant, Document.document_id == document_id)) | |
| if previous and (previous.base_id, previous.owner, previous.workspace) != (base_id, owner, workspace): | |
| raise KnowledgeError(409, "DOCUMENT_SCOPE_CONFLICT", "文档新版本不能改变授权归属。") | |
| if old is not None: | |
| if (old.sha256, old.base_id, old.owner, old.workspace) != (sha, base_id, owner, workspace): | |
| raise KnowledgeError(409, "DOCUMENT_VERSION_CONFLICT", "同一文档版本不可替换内容或归属。") | |
| self._info(db, old, filename, content, chunks) | |
| if provenance is not None: | |
| source=db.get(DocumentSource,key) | |
| if source:source.body={**provenance,"content_sha256":sha,"sha256":sha} | |
| else:db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha})) | |
| return old | |
| doc = Document(id=key, tenant=tenant, document_id=document_id, version=version, | |
| base_id=base_id, title=title, owner=owner, workspace=workspace, source_type=source_type, | |
| sha256=sha, status="failed" if error else "ready", error_code=error, | |
| valid_from=valid_from, valid_to=valid_to, | |
| created_at=datetime.now(timezone.utc).isoformat()) | |
| db.add(doc) | |
| db.flush() | |
| self._info(db, doc, filename, content, chunks) | |
| if provenance is not None: | |
| db.add(DocumentSource(document_key=key,body={**provenance,"content_sha256":sha,"sha256":sha})) | |
| for ordinal, (locator, text, terms) in enumerate(chunks, 1): | |
| db.add(Chunk(id="cite-" + digest(key + ":" + str(ordinal)), document_key=key, | |
| ordinal=ordinal, locator=locator, text=text, terms=terms)) | |
| return doc | |
| def consume_event(self, envelope): | |
| if envelope["event_type"] != "document.ingest.requested": | |
| return False | |
| user = {"tenant_id": envelope["tenant_id"], "user_id": envelope["owner_user_id"]} | |
| aid, wid = envelope["payload"]["attachment_id"], envelope["workspace_id"] | |
| info = self.workspace.attachment_info(user, aid) | |
| if info["workspace_id"] != wid: | |
| raise missing() | |
| with self.Session() as db: | |
| if "kb-personal" not in {b.id for b in self._bases(db, user)}: | |
| raise missing() | |
| doc = self._save(tenant=user["tenant_id"], document_id=aid, version="1", base_id="kb-personal", | |
| title=info["filename"], filename=info["filename"], content=self.workspace.attachment_content(user, aid), | |
| owner=user["user_id"], workspace=wid) | |
| # DB/storage exceptions propagate so the durable outbox remains pending for retry. | |
| self.workspace.apply_ingest_result(user, aid, doc.status, doc.document_id, doc.error_code) | |
| return True | |
| def import_seeds(self, private_mapping=None): | |
| """Trusted local operator entry: private_mapping maps demo key to real principal+workspace.""" | |
| manifest = json.loads((SEEDS / "manifest.json").read_text(encoding="utf-8-sig")) | |
| imported = [] | |
| for meta in manifest["documents"]: | |
| owner, wid = None, None | |
| if meta["owner_user_id"]: | |
| mapping = (private_mapping or {}).get(meta["workspace_key"]) | |
| if mapping is None: | |
| continue | |
| user, wid = mapping["principal"], mapping["workspace_id"] | |
| self.workspace.assert_access(user, wid) | |
| if user["tenant_id"] != "demo-tenant": | |
| raise missing() | |
| owner = user["user_id"] | |
| content = (SEEDS / meta["file"]).read_bytes().replace(b"\r\n", b"\n") | |
| if hashlib.sha256(content).hexdigest() != meta["sha256"]: | |
| raise KnowledgeError(422, "SEED_HASH_MISMATCH", "种子资料校验失败。") | |
| doc = self._save(tenant="demo-tenant", document_id=meta["document_id"], version=meta["version"], | |
| base_id=ALIASES[meta["kb_id"]], title=meta["title"], content=content, filename=meta["file"], | |
| owner=owner, workspace=wid, source_type=meta["source_type"], | |
| valid_from=meta["valid_from"], valid_to=meta["valid_to"]) | |
| imported.append({"document_id": doc.document_id, "status": doc.status}) | |
| return imported | |