vibeCodingP2 / experiments /knowledge_token_benchmark.py
dellweb3's picture
Upload folder using huggingface_hub
9acd20c verified
Raw History Blame Contribute Delete
5.73 kB
import json
import tempfile
import time
from pathlib import Path
import gzip
import hashlib
from src.adapters.indexes.knowledge_sqlite import SqliteKnowledgeRepository
from src.modules.knowledge.application.catalog import KnowledgeCatalog
from src.modules.skill.domain.models import Actor
from src.modules.knowledge.tests.test_knowledge import write_fixture
BUDGET_BYTES = 8000
QUERY = "Java ERP 采购订单审批"
actor = Actor(
tenant_id="test-tenant",
user_id="developer",
roles=[],
)
def size_bytes(record):
return len(str(record).encode("utf-8"))
def original_break(records):
selected = []
used = 0
for record in records:
size = size_bytes(record)
if used + size > BUDGET_BYTES:
break
selected.append(record)
used += size
return selected
def improved_continue(records):
selected = []
used = 0
for record in records:
size = size_bytes(record)
if used + size > BUDGET_BYTES:
continue
selected.append(record)
used += size
return selected
def top2(records):
return improved_continue(records[:2])
def measure(name, fn, records):
start = time.perf_counter()
selected = fn(records)
elapsed_ms = (time.perf_counter() - start) * 1000
print(f"\n[{name}]")
print("保留文档:", [r["citation_id"] for r in selected])
print("总字节:", sum(size_bytes(r) for r in selected))
print("处理耗时(ms):", round(elapsed_ms, 4))
def main():
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
repo = SqliteKnowledgeRepository(root / "index.sqlite")
corpus_dir = root / "corpus"
catalog_path = write_benchmark_corpus(corpus_dir)
repo.import_corpus(
corpus_dir,
actor.tenant_id,
catalog_path,
)
catalog = KnowledgeCatalog(repo)
start = time.perf_counter()
records = catalog.search(
actor,
QUERY,
["fixture-java"],
limit=8,
)
retrieval_ms = (time.perf_counter() - start) * 1000
print("KnowledgeCatalog 检索文档数:", len(records))
print("检索耗时(ms):", round(retrieval_ms, 3))
measure("A 原版 break", original_break, records)
measure("B 改进 continue", improved_continue, records)
measure("C Top-2", top2, records)
if len(records) < 3:
print(
"\n注意:测试库文档不足,暂时无法有效比较筛选效果。"
)
def write_benchmark_corpus(folder):
"""生成 8 条用于测试检索和预算筛选的 Java 企业知识。"""
folder.mkdir(parents=True, exist_ok=True)
specs = [
("java-style", "Java 编码规范",
"Java 类名使用 PascalCase,方法名使用 camelCase。"),
("erp-order", "ERP 采购订单流程",
"采购订单应包含供应商、商品明细、数量和金额。"),
("wms-stock", "WMS 库存流水",
"库存流水记录商品入库、出库及剩余库存数量。"),
("erp-approval", "ERP 采购审批核心规则",
"ERP 采购订单提交后必须经过审批,审批通过后才能执行采购操作。"),
("spring-api", "Spring Boot REST API",
"Spring Boot 使用 REST Controller 实现 Java HTTP API。"),
("transaction", "Java 数据库事务",
"Java 事务处理需要保证提交和回滚的一致性。"),
("maintenance", "设备维修管理",
"设备维修管理记录工单、检修状态和故障信息。"),
("java-test", "Java 单元测试规范",
"Java 单元测试通过 JUnit 验证接口行为。"),
]
records = []
for index, (record_id, title, content) in enumerate(specs):
content = content * (40 if record_id == "erp-order" else 5)
record = {
"record_id": record_id,
"collection_id": "fixture-java",
"title": title,
"content": content,
"content_sha256": hashlib.sha256(content.encode()).hexdigest(),
"source_url": f"https://example.com/{record_id}",
"source_path": f"{record_id}.java",
"source_commit": "a" * 40,
"line_start": 1,
"line_end": 10,
"license": "Apache-2.0",
"usage_policy": "attribution-required",
"quality_status": "imported_unreviewed",
"trust": "external_reference",
}
records.append(record)
with gzip.open(
folder / "fixture-java.jsonl.gz", "wt", encoding="utf-8"
) as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
catalog = folder / "catalog.json"
# 生成符合原项目导入格式的 manifest.json
data_file = folder / "fixture-java.jsonl.gz"
manifest = {
"collections": [
{
"id": "fixture-java",
"name": "Java ERP 测试知识库",
"description": "Token optimization benchmark",
"sha256": hashlib.sha256(
data_file.read_bytes()
).hexdigest(),
"records": len(records),
"encoding": "utf-8",
}
]
}
(folder / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False),
encoding="utf-8",
)
catalog.write_text(
json.dumps(
{"collections": [{"id": "fixture-java"}]},
ensure_ascii=False,
),
encoding="utf-8",
)
return catalog
if __name__ == "__main__":
main()