File size: 3,499 Bytes
9acd20c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
import time
from sentence_transformers import SentenceTransformer

# 实验参数
BUDGET_BYTES = 8000
MODEL_NAME = "intfloat/multilingual-e5-small"

query = "如何实现 Java ERP 采购订单审批接口?"

# 模拟知识库返回顺序
documents = [
    {
        "id": "A",
        "title": "Java 开发规范",
        "text": "Java 项目应使用清晰的类名和方法名。" * 65,
        "required": False,
    },
    {
        "id": "B",
        "title": "ERP 采购基础介绍",
        "text": "ERP 系统用于管理采购订单及供应商信息。" * 55,
        "required": False,
    },
    {
        "id": "C",
        "title": "较长的采购背景资料",
        "text": "企业采购业务包含多个管理流程和操作步骤。" * 85,
        "required": False,
    },
    {
        "id": "D",
        "title": "采购审批核心规则",
        "text": (
            "采购订单提交后必须经过审批,"
            "审批通过后才能执行后续采购操作。"
        ),
        "required": True,
    },
]


def size_bytes(doc):
    return len(str(doc).encode("utf-8"))


# A:原版遇到超限直接结束
def select_break(docs):
    selected = []
    used = 0

    for doc in docs:
        size = size_bytes(doc)

        if used + size > BUDGET_BYTES:
            break

        selected.append(doc)
        used += size

    return selected


# B:跳过过长文档,继续检查
def select_continue(docs):
    selected = []
    used = 0

    for doc in docs:
        size = size_bytes(doc)

        if used + size > BUDGET_BYTES:
            continue

        selected.append(doc)
        used += size

    return selected


# 加载你已经下载的 Embedding 模型
print("加载语义模型...")
model = SentenceTransformer(MODEL_NAME)

# 预先计算文档向量
passages = [
    "passage: " + doc["title"] + " " + doc["text"]
    for doc in documents
]
doc_vectors = model.encode(
    passages,
    normalize_embeddings=True,
)


# C:语义重排序 + 预算
def select_semantic(docs):
    query_vector = model.encode(
        "query: " + query,
        normalize_embeddings=True,
    )

    scores = doc_vectors @ query_vector

    ranked = sorted(
        zip(docs, scores),
        key=lambda item: float(item[1]),
        reverse=True,
    )

    selected = []
    used = 0

    for doc, score in ranked:
        size = size_bytes(doc)

        if used + size > BUDGET_BYTES:
            continue

        selected.append(doc)
        used += size

    return selected


def evaluate(name, func):
    start = time.perf_counter()
    selected = func(documents)
    elapsed = (time.perf_counter() - start) * 1000

    used = sum(size_bytes(doc) for doc in selected)
    has_required = any(
        doc["required"] for doc in selected
    )

    print(f"\n【{name}】")
    print("保留文档:", [d["id"] for d in selected])
    print("使用字节:", used)
    print("核心审批规则是否保留:", has_required)
    print("处理耗时(ms):", round(elapsed, 3))


evaluate("A 原版 break", select_break)
evaluate("B 改进 continue", select_continue)
evaluate("C 语义重排序", select_semantic)

# D:直接保留前两条检索结果
def select_top2(docs):
    selected = []
    used = 0

    for doc in docs[:2]:
        size = size_bytes(doc)

        if used + size > BUDGET_BYTES:
            continue

        selected.append(doc)
        used += size

    return selected


evaluate("D Top-2 精简", select_top2)