import re from sentence_transformers import SentenceTransformer, util # 1. 使用已经下载过的语义模型 MODEL_NAME = "intfloat/multilingual-e5-small" # 2. 模拟企业知识库(每份文档都有 ID) documents = [ { "id": "WMS-001", "text": "库存管理系统记录商品入库、出库和剩余库存数量。", }, { "id": "ERP-001", "text": "ERP 采购订单系统负责供应商报价、采购审批和付款流程。", }, { "id": "JAVA-001", "text": "Java Spring Boot 通过 REST Controller 实现 HTTP API。", }, { "id": "DEVICE-001", "text": "设备运维系统负责设备巡检、维修工单与故障记录。", }, { "id": "JAVA-002", "text": "Java 中的 inventoryTransaction 用于库存事务处理。", }, ] # 3. 提取检索词:中文双字组合 + 英文单词 def tokenize(text): text = text.lower() tokens = set(re.findall(r"[a-z][a-z0-9_]*", text)) for segment in re.findall(r"[\u4e00-\u9fff]+", text): for i in range(len(segment) - 1): tokens.add(segment[i:i + 2]) return tokens # 4. 关键词检索(简化实验版) def keyword_search(query): query_tokens = tokenize(query) results = [] for doc in documents: doc_tokens = tokenize(doc["text"]) score = len(query_tokens & doc_tokens) if score > 0: results.append((doc["id"], score)) return sorted( results, key=lambda item: item[1], reverse=True, ) # 5. 加载语义模型 print("正在加载 Embedding 模型...") model = SentenceTransformer(MODEL_NAME) passages = [ "passage: " + doc["text"] for doc in documents ] doc_vectors = model.encode( passages, convert_to_tensor=True, normalize_embeddings=True, ) # 6. 语义检索 def semantic_search(query): query_vector = model.encode( "query: " + query, convert_to_tensor=True, normalize_embeddings=True, ) similarities = util.cos_sim( query_vector, doc_vectors )[0] results = [ (doc["id"], float(similarities[i])) for i, doc in enumerate(documents) ] return sorted( results, key=lambda item: item[1], reverse=True, ) # 7. 混合检索:按排名融合 def hybrid_search(query): keyword_results = keyword_search(query) semantic_results = semantic_search(query) # RRF:排名越靠前,得分越高 scores = {} k = 60 for rank, (doc_id, _) in enumerate( keyword_results, start=1 ): scores[doc_id] = ( scores.get(doc_id, 0) + 1 / (k + rank) ) for rank, (doc_id, _) in enumerate( semantic_results, start=1 ): scores[doc_id] = ( scores.get(doc_id, 0) + 1 / (k + rank) ) results = sorted( scores.items(), key=lambda item: item[1], reverse=True, ) return keyword_results, semantic_results, results # 8. 展示搜索结果 def display(query): kw, sem, hybrid = hybrid_search(query) print("\n" + "=" * 55) print("用户问题:", query) print("\n【关键词检索 Top 3】") for doc_id, score in kw[:3]: print(doc_id, "词项得分:", score) print("\n【语义检索 Top 3】") for doc_id, score in sem[:3]: print(doc_id, "相似度:", round(score, 4)) print("\n【混合检索 Top 3】") for doc_id, score in hybrid[:3]: doc = next( d for d in documents if d["id"] == doc_id ) print(doc_id, "RRF:", round(score, 5)) print(" 文档:", doc["text"]) # 9. 测试三个场景 questions = [ "如何追踪商品的进出及剩余数量?", "怎样管理供应商报价和采购审批?", "Spring Boot 如何编写 REST API?", ] for question in questions: display(question)