Spaces:
Sleeping
Sleeping
File size: 8,560 Bytes
caf8059 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | """Local lexical index with citations and tenant ACL. No embeddings are required."""
from __future__ import annotations
import gzip
import hashlib
import json
import re
import sqlite3
from contextlib import contextmanager
from pathlib import Path
from src.modules.skill.domain.models import SkillError
ALIASES = {"库存": "inventory stock warehouse", "仓库": "warehouse inventory", "批次": "batch lot", "出库": "shipment delivery inventory", "订单": "order purchase", "审批": "approval workflow", "金额": "amount decimal", "设备": "device asset", "运维": "maintenance alarm", "维修": "maintenance work order", "巡检": "inspection schedule", "集合": "collection list map", "并发": "concurrent thread lock", "事务": "transaction commit rollback", "设计模式": "pattern factory strategy", "单元测试": "test assert mock", "页面": "html element document", "认证": "authentication authorization"}
class SqliteKnowledgeRepository:
def __init__(self, path):
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
with self.connect() as db:
db.executescript("""
CREATE TABLE IF NOT EXISTS kb_collections(tenant TEXT NOT NULL,collection_id TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(tenant,collection_id));
CREATE TABLE IF NOT EXISTS kb_records(tenant TEXT NOT NULL,collection_id TEXT NOT NULL,record_id TEXT NOT NULL,content_sha256 TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(tenant,record_id),UNIQUE(tenant,content_sha256));
CREATE INDEX IF NOT EXISTS kb_collection_idx ON kb_records(tenant,collection_id);
CREATE VIRTUAL TABLE IF NOT EXISTS kb_fts USING fts5(tenant UNINDEXED,collection_id UNINDEXED,record_id UNINDEXED,title,content,tokenize='unicode61');
CREATE TABLE IF NOT EXISTS kb_audit(id INTEGER PRIMARY KEY,tenant TEXT NOT NULL,user_id TEXT NOT NULL,query_digest TEXT NOT NULL,collections TEXT NOT NULL,result_count INTEGER NOT NULL);
""")
@contextmanager
def connect(self):
db = sqlite3.connect(self.path, timeout=60)
db.row_factory = sqlite3.Row
try:
yield db
db.commit()
except BaseException:
db.rollback()
raise
finally:
db.close()
def import_corpus(self, directory, tenant, catalog_path):
"""Trusted local ingestion entry point; unavailable as arbitrary path API."""
directory = Path(directory)
manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
allowed = {c["id"] for c in json.loads(Path(catalog_path).read_text(encoding="utf-8"))["collections"]}
if {c["id"] for c in manifest["collections"]} != allowed:
raise ValueError("CORPUS_CATALOG_MISMATCH")
result = []
for entry in manifest["collections"]:
cid = entry["id"]
path = directory / (cid + ".jsonl.gz")
if hashlib.sha256(path.read_bytes()).hexdigest() != entry["sha256"]:
raise ValueError("CORPUS_DIGEST_MISMATCH")
with self.connect() as db:
prior = db.execute("SELECT body FROM kb_collections WHERE tenant=? AND collection_id=?", (tenant, cid)).fetchone()
revision = "corpus-" + entry["sha256"][:24]
if prior:
old = json.loads(prior["body"])
if old["revision"] == revision:
result.append(old)
continue
raise ValueError("IMMUTABLE_CORPUS_REVISION_EXISTS: import a new index/profile instead of replacing running references")
count = 0
with gzip.open(path, "rt", encoding="utf-8") as gz:
for line in gz:
if len(line) > 40000:
raise ValueError("CORPUS_RECORD_TOO_LARGE")
record = json.loads(line)
content = record["content"]
if record["collection_id"] != cid or not content or len(content) < 280 or record["quality_status"] != "imported_unreviewed" or not re.fullmatch(r"[a-f0-9]{40}", record["source_commit"]):
raise ValueError("CORPUS_RECORD_INVALID")
if hashlib.sha256(re.sub(r"\s+", " ", content).strip().encode()).hexdigest() != record["content_sha256"]:
raise ValueError("CORPUS_CONTENT_DIGEST_MISMATCH")
db.execute("INSERT INTO kb_records VALUES(?,?,?,?,?)", (tenant, cid, record["record_id"], record["content_sha256"], json.dumps(record, ensure_ascii=False)))
db.execute("INSERT INTO kb_fts VALUES(?,?,?,?,?)", (tenant, cid, record["record_id"], record["title"], content))
count += 1
if count != entry["records"]:
raise ValueError("CORPUS_COUNT_MISMATCH")
body = {"collection_id": cid, "name": entry["name"], "description": entry["description"], "revision": revision, "digest": entry["sha256"], "record_count": count, "quality_status": "imported_unreviewed", "enterprise_approved": False, "status": "reference_ready", "visibility": "tenant", "read_users": [], "schema": "zkcoding.knowledge-collection/v1"}
db.execute("INSERT INTO kb_collections VALUES(?,?,?)", (tenant, cid, json.dumps(body, ensure_ascii=False)))
result.append(body)
return result
@staticmethod
def readable(actor, body):
return body["visibility"] == "tenant" or actor.user_id in body.get("read_users", []) or "admin" in actor.roles
def collection(self, actor, cid):
with self.connect() as db:
row = db.execute("SELECT body FROM kb_collections WHERE tenant=? AND collection_id=?", (actor.tenant_id, cid)).fetchone()
body = json.loads(row["body"]) if row else None
if body is None or not self.readable(actor, body):
raise SkillError("KNOWLEDGE_NOT_FOUND", "知识库不存在或不可见。", 404)
return body
def collections(self, actor):
with self.connect() as db:
bodies = [json.loads(row["body"]) for row in db.execute("SELECT body FROM kb_collections WHERE tenant=? ORDER BY collection_id", (actor.tenant_id,))]
return [b for b in bodies if self.readable(actor, b)]
def search(self, actor, query, collection_ids, limit, *, for_code_reuse=False):
expanded = query + " " + " ".join(en for zh, en in ALIASES.items() if zh in query)
terms = list(dict.fromkeys(re.findall(r"[A-Za-z0-9_]{2,}|[\u4e00-\u9fff]+", expanded)))[:24]
if not terms:
return []
# Prefix matching makes Java identifiers such as inventoryTransaction discoverable.
# This remains lexical search, not semantic segmentation; quote every user-derived term.
expression = " OR ".join('"' + t + '"' + ('*' if re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]{2,}', t) else '') for t in terms)
placeholders = ",".join("?" for _ in collection_ids)
with self.connect() as db:
# Fetch a bounded surplus so restricted code-reuse licenses can be excluded.
rows = db.execute(f"SELECT r.body FROM kb_fts JOIN kb_records r ON r.tenant=kb_fts.tenant AND r.record_id=kb_fts.record_id WHERE kb_fts MATCH ? AND kb_fts.tenant=? AND kb_fts.collection_id IN ({placeholders}) ORDER BY bm25(kb_fts),kb_fts.record_id LIMIT ?", (expression, actor.tenant_id, *collection_ids, min(200, limit * 10))).fetchall()
records = [json.loads(row["body"]) for row in rows]
if for_code_reuse:
records = [r for r in records if r["usage_policy"] == "attribution-required" and r["license"] in {"Apache-2.0", "MIT"}]
records = records[:limit]
db.execute("INSERT INTO kb_audit(tenant,user_id,query_digest,collections,result_count) VALUES(?,?,?,?,?)", (actor.tenant_id, actor.user_id, hashlib.sha256(query.encode()).hexdigest(), json.dumps(collection_ids), len(records)))
return [{"citation_id": r["record_id"], "collection_id": r["collection_id"], "title": r["title"], "snippet": r["content"], "source_url": r["source_url"], "source_commit": r["source_commit"], "line_start": r["line_start"], "line_end": r["line_end"], "content_sha256": r["content_sha256"], "license": r["license"], "usage_policy": r["usage_policy"], "quality_status": r["quality_status"], "trust": r["trust"]} for r in records]
|