vibeCodingP2 / experiments /token_benchmark.py
dellweb3's picture
Upload folder using huggingface_hub
9acd20c verified
Raw History Blame Contribute Delete
3.5 kB
import time
from sentence_transformers import SentenceTransformer
# 实验参数
BUDGET_BYTES = 8000
MODEL_NAME = "intfloat/multilingual-e5-small"
query = "如何实现 Java ERP 采购订单审批接口?"
# 模拟知识库返回顺序
documents = [
{
"id": "A",
"title": "Java 开发规范",
"text": "Java 项目应使用清晰的类名和方法名。" * 65,
"required": False,
},
{
"id": "B",
"title": "ERP 采购基础介绍",
"text": "ERP 系统用于管理采购订单及供应商信息。" * 55,
"required": False,
},
{
"id": "C",
"title": "较长的采购背景资料",
"text": "企业采购业务包含多个管理流程和操作步骤。" * 85,
"required": False,
},
{
"id": "D",
"title": "采购审批核心规则",
"text": (
"采购订单提交后必须经过审批,"
"审批通过后才能执行后续采购操作。"
),
"required": True,
},
]
def size_bytes(doc):
return len(str(doc).encode("utf-8"))
# A:原版遇到超限直接结束
def select_break(docs):
selected = []
used = 0
for doc in docs:
size = size_bytes(doc)
if used + size > BUDGET_BYTES:
break
selected.append(doc)
used += size
return selected
# B:跳过过长文档,继续检查
def select_continue(docs):
selected = []
used = 0
for doc in docs:
size = size_bytes(doc)
if used + size > BUDGET_BYTES:
continue
selected.append(doc)
used += size
return selected
# 加载你已经下载的 Embedding 模型
print("加载语义模型...")
model = SentenceTransformer(MODEL_NAME)
# 预先计算文档向量
passages = [
"passage: " + doc["title"] + " " + doc["text"]
for doc in documents
]
doc_vectors = model.encode(
passages,
normalize_embeddings=True,
)
# C:语义重排序 + 预算
def select_semantic(docs):
query_vector = model.encode(
"query: " + query,
normalize_embeddings=True,
)
scores = doc_vectors @ query_vector
ranked = sorted(
zip(docs, scores),
key=lambda item: float(item[1]),
reverse=True,
)
selected = []
used = 0
for doc, score in ranked:
size = size_bytes(doc)
if used + size > BUDGET_BYTES:
continue
selected.append(doc)
used += size
return selected
def evaluate(name, func):
start = time.perf_counter()
selected = func(documents)
elapsed = (time.perf_counter() - start) * 1000
used = sum(size_bytes(doc) for doc in selected)
has_required = any(
doc["required"] for doc in selected
)
print(f"\n【{name}】")
print("保留文档:", [d["id"] for d in selected])
print("使用字节:", used)
print("核心审批规则是否保留:", has_required)
print("处理耗时(ms):", round(elapsed, 3))
evaluate("A 原版 break", select_break)
evaluate("B 改进 continue", select_continue)
evaluate("C 语义重排序", select_semantic)
# D:直接保留前两条检索结果
def select_top2(docs):
selected = []
used = 0
for doc in docs[:2]:
size = size_bytes(doc)
if used + size > BUDGET_BYTES:
continue
selected.append(doc)
used += size
return selected
evaluate("D Top-2 精简", select_top2)