vibeCodingP2 / experiments /hybrid_search.py
dellweb3's picture
Add Knowledge routing and semantic search experiments
2f6097e verified
Raw History Blame Contribute Delete
3.96 kB
import re
from sentence_transformers import SentenceTransformer, util
# 1. 使用已经下载过的语义模型
MODEL_NAME = "intfloat/multilingual-e5-small"
# 2. 模拟企业知识库(每份文档都有 ID)
documents = [
{
"id": "WMS-001",
"text": "库存管理系统记录商品入库、出库和剩余库存数量。",
},
{
"id": "ERP-001",
"text": "ERP 采购订单系统负责供应商报价、采购审批和付款流程。",
},
{
"id": "JAVA-001",
"text": "Java Spring Boot 通过 REST Controller 实现 HTTP API。",
},
{
"id": "DEVICE-001",
"text": "设备运维系统负责设备巡检、维修工单与故障记录。",
},
{
"id": "JAVA-002",
"text": "Java 中的 inventoryTransaction 用于库存事务处理。",
},
]
# 3. 提取检索词:中文双字组合 + 英文单词
def tokenize(text):
text = text.lower()
tokens = set(re.findall(r"[a-z][a-z0-9_]*", text))
for segment in re.findall(r"[\u4e00-\u9fff]+", text):
for i in range(len(segment) - 1):
tokens.add(segment[i:i + 2])
return tokens
# 4. 关键词检索(简化实验版)
def keyword_search(query):
query_tokens = tokenize(query)
results = []
for doc in documents:
doc_tokens = tokenize(doc["text"])
score = len(query_tokens & doc_tokens)
if score > 0:
results.append((doc["id"], score))
return sorted(
results,
key=lambda item: item[1],
reverse=True,
)
# 5. 加载语义模型
print("正在加载 Embedding 模型...")
model = SentenceTransformer(MODEL_NAME)
passages = [
"passage: " + doc["text"]
for doc in documents
]
doc_vectors = model.encode(
passages,
convert_to_tensor=True,
normalize_embeddings=True,
)
# 6. 语义检索
def semantic_search(query):
query_vector = model.encode(
"query: " + query,
convert_to_tensor=True,
normalize_embeddings=True,
)
similarities = util.cos_sim(
query_vector, doc_vectors
)[0]
results = [
(doc["id"], float(similarities[i]))
for i, doc in enumerate(documents)
]
return sorted(
results,
key=lambda item: item[1],
reverse=True,
)
# 7. 混合检索:按排名融合
def hybrid_search(query):
keyword_results = keyword_search(query)
semantic_results = semantic_search(query)
# RRF:排名越靠前,得分越高
scores = {}
k = 60
for rank, (doc_id, _) in enumerate(
keyword_results, start=1
):
scores[doc_id] = (
scores.get(doc_id, 0) + 1 / (k + rank)
)
for rank, (doc_id, _) in enumerate(
semantic_results, start=1
):
scores[doc_id] = (
scores.get(doc_id, 0) + 1 / (k + rank)
)
results = sorted(
scores.items(),
key=lambda item: item[1],
reverse=True,
)
return keyword_results, semantic_results, results
# 8. 展示搜索结果
def display(query):
kw, sem, hybrid = hybrid_search(query)
print("\n" + "=" * 55)
print("用户问题:", query)
print("\n【关键词检索 Top 3】")
for doc_id, score in kw[:3]:
print(doc_id, "词项得分:", score)
print("\n【语义检索 Top 3】")
for doc_id, score in sem[:3]:
print(doc_id, "相似度:", round(score, 4))
print("\n【混合检索 Top 3】")
for doc_id, score in hybrid[:3]:
doc = next(
d for d in documents if d["id"] == doc_id
)
print(doc_id, "RRF:", round(score, 5))
print(" 文档:", doc["text"])
# 9. 测试三个场景
questions = [
"如何追踪商品的进出及剩余数量?",
"怎样管理供应商报价和采购审批?",
"Spring Boot 如何编写 REST API?",
]
for question in questions:
display(question)