Spaces:
Paused
Paused
Download experiments/hybrid_search.py from dellweb3/vibeCodingP2: direct link, hf CLI and curl.
- Browser
- Download file 3.96 kB
-
https://huggingface.co/spaces/dellweb3/vibeCodingP2/resolve/main/experiments/hybrid_search.py
- Command line
-
hf download hf://spaces/dellweb3/vibeCodingP2/experiments/hybrid_search.py
-
curl -L -o hybrid_search.py https://huggingface.co/spaces/dellweb3/vibeCodingP2/resolve/main/experiments/hybrid_search.py
3.96 kB
| import re | |
| from sentence_transformers import SentenceTransformer, util | |
| # 1. 使用已经下载过的语义模型 | |
| MODEL_NAME = "intfloat/multilingual-e5-small" | |
| # 2. 模拟企业知识库(每份文档都有 ID) | |
| documents = [ | |
| { | |
| "id": "WMS-001", | |
| "text": "库存管理系统记录商品入库、出库和剩余库存数量。", | |
| }, | |
| { | |
| "id": "ERP-001", | |
| "text": "ERP 采购订单系统负责供应商报价、采购审批和付款流程。", | |
| }, | |
| { | |
| "id": "JAVA-001", | |
| "text": "Java Spring Boot 通过 REST Controller 实现 HTTP API。", | |
| }, | |
| { | |
| "id": "DEVICE-001", | |
| "text": "设备运维系统负责设备巡检、维修工单与故障记录。", | |
| }, | |
| { | |
| "id": "JAVA-002", | |
| "text": "Java 中的 inventoryTransaction 用于库存事务处理。", | |
| }, | |
| ] | |
| # 3. 提取检索词:中文双字组合 + 英文单词 | |
| def tokenize(text): | |
| text = text.lower() | |
| tokens = set(re.findall(r"[a-z][a-z0-9_]*", text)) | |
| for segment in re.findall(r"[\u4e00-\u9fff]+", text): | |
| for i in range(len(segment) - 1): | |
| tokens.add(segment[i:i + 2]) | |
| return tokens | |
| # 4. 关键词检索(简化实验版) | |
| def keyword_search(query): | |
| query_tokens = tokenize(query) | |
| results = [] | |
| for doc in documents: | |
| doc_tokens = tokenize(doc["text"]) | |
| score = len(query_tokens & doc_tokens) | |
| if score > 0: | |
| results.append((doc["id"], score)) | |
| return sorted( | |
| results, | |
| key=lambda item: item[1], | |
| reverse=True, | |
| ) | |
| # 5. 加载语义模型 | |
| print("正在加载 Embedding 模型...") | |
| model = SentenceTransformer(MODEL_NAME) | |
| passages = [ | |
| "passage: " + doc["text"] | |
| for doc in documents | |
| ] | |
| doc_vectors = model.encode( | |
| passages, | |
| convert_to_tensor=True, | |
| normalize_embeddings=True, | |
| ) | |
| # 6. 语义检索 | |
| def semantic_search(query): | |
| query_vector = model.encode( | |
| "query: " + query, | |
| convert_to_tensor=True, | |
| normalize_embeddings=True, | |
| ) | |
| similarities = util.cos_sim( | |
| query_vector, doc_vectors | |
| )[0] | |
| results = [ | |
| (doc["id"], float(similarities[i])) | |
| for i, doc in enumerate(documents) | |
| ] | |
| return sorted( | |
| results, | |
| key=lambda item: item[1], | |
| reverse=True, | |
| ) | |
| # 7. 混合检索:按排名融合 | |
| def hybrid_search(query): | |
| keyword_results = keyword_search(query) | |
| semantic_results = semantic_search(query) | |
| # RRF:排名越靠前,得分越高 | |
| scores = {} | |
| k = 60 | |
| for rank, (doc_id, _) in enumerate( | |
| keyword_results, start=1 | |
| ): | |
| scores[doc_id] = ( | |
| scores.get(doc_id, 0) + 1 / (k + rank) | |
| ) | |
| for rank, (doc_id, _) in enumerate( | |
| semantic_results, start=1 | |
| ): | |
| scores[doc_id] = ( | |
| scores.get(doc_id, 0) + 1 / (k + rank) | |
| ) | |
| results = sorted( | |
| scores.items(), | |
| key=lambda item: item[1], | |
| reverse=True, | |
| ) | |
| return keyword_results, semantic_results, results | |
| # 8. 展示搜索结果 | |
| def display(query): | |
| kw, sem, hybrid = hybrid_search(query) | |
| print("\n" + "=" * 55) | |
| print("用户问题:", query) | |
| print("\n【关键词检索 Top 3】") | |
| for doc_id, score in kw[:3]: | |
| print(doc_id, "词项得分:", score) | |
| print("\n【语义检索 Top 3】") | |
| for doc_id, score in sem[:3]: | |
| print(doc_id, "相似度:", round(score, 4)) | |
| print("\n【混合检索 Top 3】") | |
| for doc_id, score in hybrid[:3]: | |
| doc = next( | |
| d for d in documents if d["id"] == doc_id | |
| ) | |
| print(doc_id, "RRF:", round(score, 5)) | |
| print(" 文档:", doc["text"]) | |
| # 9. 测试三个场景 | |
| questions = [ | |
| "如何追踪商品的进出及剩余数量?", | |
| "怎样管理供应商报价和采购审批?", | |
| "Spring Boot 如何编写 REST API?", | |
| ] | |
| for question in questions: | |
| display(question) | |