Spaces:
Sleeping
Sleeping
Download KnowledgeService/semantic.py from dellweb3/workos-office: direct link, hf CLI and curl.
- Browser
- Download file 2.22 kB
-
https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/semantic.py
- Command line
-
hf download hf://spaces/dellweb3/workos-office/KnowledgeService/semantic.py
-
curl -L -o semantic.py https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/KnowledgeService/semantic.py
2.22 kB
| """Local BGE embeddings over authorized chunks; explicit failure without a model.""" | |
| import os | |
| import threading | |
| from pathlib import Path | |
| from APIGateway.app.errors import GatewayError | |
| _lock = threading.RLock() | |
| _model = None | |
| _vectors = {} | |
| MODEL_ID = 'BAAI/bge-small-zh-v1.5' | |
| def model_path(): | |
| configured = os.getenv('WORKOS_EMBEDDING_MODEL_PATH') | |
| if configured: return configured | |
| homes = [Path(os.getenv('HF_HOME', str(Path.home()/'.cache/huggingface'))), | |
| Path(r'E:\codex\models\huggingface'), Path(r'D:\codex\models\huggingface')] | |
| for home in homes: | |
| for folder in sorted((home/'hub/models--BAAI--bge-small-zh-v1.5/snapshots').glob('*')): | |
| if (folder/'model.safetensors').is_file():return str(folder) | |
| return MODEL_ID | |
| def rank_semantic(query, rows, top_k): | |
| if not rows:return [] | |
| global _model | |
| with _lock: | |
| try: | |
| import numpy as np | |
| if _model is None: | |
| from sentence_transformers import SentenceTransformer | |
| _model = SentenceTransformer(model_path(), device='cpu', local_files_only=True) | |
| missing=[(c,d) for c,d in rows if c.id not in _vectors] | |
| if missing: | |
| vectors=_model.encode([c.text for c,d in missing], batch_size=32, normalize_embeddings=True, show_progress_bar=False) | |
| _vectors.update({c.id:v for (c,d),v in zip(missing,vectors)}) | |
| if len(_vectors)>10000: | |
| keep={c.id for c,d in rows};_vectors.update({}) | |
| for key in list(_vectors): | |
| if key not in keep:del _vectors[key] | |
| vector=_model.encode(['为这个句子生成表示以用于检索相关文章:'+query], normalize_embeddings=True, show_progress_bar=False)[0] | |
| scores=np.asarray([_vectors[c.id] for c,d in rows]) @ vector | |
| except (ImportError, OSError, RuntimeError) as exc: | |
| raise GatewayError(503, 'EMBEDDING_UNAVAILABLE', '本地 BGE 向量模型未准备好;请安装向量依赖并配置 WORKOS_EMBEDDING_MODEL_PATH。') from exc | |
| scored=[(float(score),c,d) for score,(c,d) in zip(scores,rows)] | |
| return sorted(scored,key=lambda x:(-x[0],x[1].id))[:top_k] | |