Download src/engine/vector_store.py from YGCXLab/ux-research-agent-slackbot: direct link, hf CLI and curl.
- Browser
- Download file 8.37 kB
-
https://huggingface.co/spaces/YGCXLab/ux-research-agent-slackbot/resolve/main/src/engine/vector_store.py
- Command line
-
hf download hf://spaces/YGCXLab/ux-research-agent-slackbot/src/engine/vector_store.py
-
curl -L -o vector_store.py https://huggingface.co/spaces/YGCXLab/ux-research-agent-slackbot/resolve/main/src/engine/vector_store.py
8.37 kB
| """ChromaDB-based vector store for semantic search. | |
| Uses Google text-embedding-004 to embed documents from all three sources | |
| (INTERNAL, BAYMARD, NNG) and provides cosine similarity search. | |
| Gracefully degrades: if initialization fails, callers fall back to fuzzy-only search. | |
| """ | |
| import hashlib | |
| import json | |
| import logging | |
| from pathlib import Path | |
| from typing import List, Dict, Any, Optional | |
| from .config import config, SCORING, PROJECT_ROOT | |
| logger = logging.getLogger("VectorStore") | |
| VECTOR_STORE_DIR = PROJECT_ROOT / ".vector_store" | |
| HASH_FILE = VECTOR_STORE_DIR / "data_hash.json" | |
| COLLECTION_NAME = "ux_research_docs" | |
| def _build_internal_text(item: Dict[str, Any], max_len: int) -> str: | |
| """Build embedding text for an INTERNAL research document.""" | |
| meta = item.get("metadata", {}) | |
| detail = item.get("상세 정보", {}) | |
| parts = [ | |
| meta.get("리서치 제목", ""), | |
| meta.get("태그", ""), | |
| meta.get("도메인", ""), | |
| meta.get("방법론", ""), | |
| detail.get("주요 내용 요약", ""), | |
| ] | |
| return " ".join(p for p in parts if p)[:max_len] | |
| def _build_baymard_text(item: Dict[str, Any], max_len: int) -> str: | |
| """Build embedding text for a BAYMARD guideline.""" | |
| parts = [ | |
| item.get("title", ""), | |
| item.get("topic", ""), | |
| item.get("issue", ""), | |
| item.get("advice", ""), | |
| ] | |
| return " ".join(p for p in parts if p)[:max_len] | |
| def _build_nng_text(item: Dict[str, Any], max_len: int) -> str: | |
| """Build embedding text for a NNG article.""" | |
| contents = item.get("contents", []) | |
| contents_str = " ".join(contents) if isinstance(contents, list) else str(contents) | |
| parts = [ | |
| item.get("title", ""), | |
| item.get("description", ""), | |
| " ".join(item.get("topics", [])), | |
| contents_str, | |
| ] | |
| return " ".join(p for p in parts if p)[:max_len] | |
| class VectorStore: | |
| """ChromaDB vector store with Google Generative AI embeddings.""" | |
| def __init__( | |
| self, | |
| internal_data: List[Dict], | |
| baymard_data: List[Dict], | |
| nng_data: List[Dict], | |
| ): | |
| self.internal_data = internal_data | |
| self.baymard_data = baymard_data | |
| self.nng_data = nng_data | |
| self.max_len = SCORING.max_embedding_text_length | |
| self.batch_size = SCORING.embedding_batch_size | |
| self._collection = None | |
| self._initialize() | |
| def _compute_data_hash(self) -> str: | |
| """Compute a SHA-256 hash from document counts and boundary IDs.""" | |
| parts = [] | |
| for label, data, id_fn in [ | |
| ("internal", self.internal_data, lambda d: d.get("id", "")), | |
| ("baymard", self.baymard_data, lambda d: str(d.get("id", ""))), | |
| ("nng", self.nng_data, lambda d: d.get("id", "")), | |
| ]: | |
| count = len(data) | |
| first_id = id_fn(data[0]) if data else "" | |
| last_id = id_fn(data[-1]) if data else "" | |
| parts.append(f"{label}:{count}:{first_id}:{last_id}") | |
| raw = "|".join(parts) | |
| return hashlib.sha256(raw.encode()).hexdigest() | |
| def _initialize(self): | |
| """Load existing index or rebuild if data hash changed.""" | |
| import chromadb | |
| from langchain_google_genai import GoogleGenerativeAIEmbeddings | |
| self._embedding_fn = GoogleGenerativeAIEmbeddings( | |
| model="models/text-embedding-004", | |
| google_api_key=config.google_api_key, | |
| ) | |
| VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True) | |
| self._client = chromadb.PersistentClient(path=str(VECTOR_STORE_DIR)) | |
| current_hash = self._compute_data_hash() | |
| # Check if existing index matches current data | |
| stored_hash = None | |
| if HASH_FILE.exists(): | |
| try: | |
| stored_hash = json.loads(HASH_FILE.read_text()).get("hash") | |
| except Exception: | |
| pass | |
| if stored_hash == current_hash: | |
| # Reuse existing collection | |
| try: | |
| self._collection = self._client.get_collection(name=COLLECTION_NAME) | |
| if self._collection.count() > 0: | |
| logger.info( | |
| "Loaded existing vector index (%d docs, hash=%s…)", | |
| self._collection.count(), | |
| current_hash[:8], | |
| ) | |
| return | |
| except Exception: | |
| pass | |
| # Rebuild | |
| logger.info("Building vector index (hash=%s…)…", current_hash[:8]) | |
| self._build_index() | |
| # Save hash | |
| HASH_FILE.write_text(json.dumps({"hash": current_hash})) | |
| logger.info("Vector index built and saved.") | |
| def _build_index(self): | |
| """Embed all documents and store in ChromaDB.""" | |
| # Delete old collection if exists | |
| try: | |
| self._client.delete_collection(name=COLLECTION_NAME) | |
| except Exception: | |
| pass | |
| self._collection = self._client.create_collection( | |
| name=COLLECTION_NAME, | |
| metadata={"hnsw:space": "cosine"}, | |
| ) | |
| all_docs = [] # (id, text, metadata) | |
| for item in self.internal_data: | |
| doc_id = item.get("id", "") | |
| if not doc_id: | |
| continue | |
| text = _build_internal_text(item, self.max_len) | |
| if text.strip(): | |
| all_docs.append(( | |
| f"INTERNAL:{doc_id}", | |
| text, | |
| {"source": "INTERNAL", "doc_id": doc_id}, | |
| )) | |
| for item in self.baymard_data: | |
| doc_id = str(item.get("id", "")) | |
| if not doc_id: | |
| continue | |
| text = _build_baymard_text(item, self.max_len) | |
| if text.strip(): | |
| all_docs.append(( | |
| f"BAYMARD:{doc_id}", | |
| text, | |
| {"source": "BAYMARD", "doc_id": doc_id}, | |
| )) | |
| for item in self.nng_data: | |
| doc_id = item.get("id", "") | |
| if not doc_id: | |
| continue | |
| text = _build_nng_text(item, self.max_len) | |
| if text.strip(): | |
| all_docs.append(( | |
| f"NNG:{doc_id}", | |
| text, | |
| {"source": "NNG", "doc_id": doc_id}, | |
| )) | |
| logger.info("Embedding %d documents…", len(all_docs)) | |
| # Batch embed and upsert | |
| for i in range(0, len(all_docs), self.batch_size): | |
| batch = all_docs[i : i + self.batch_size] | |
| ids = [d[0] for d in batch] | |
| texts = [d[1] for d in batch] | |
| metadatas = [d[2] for d in batch] | |
| embeddings = self._embedding_fn.embed_documents(texts) | |
| self._collection.add( | |
| ids=ids, | |
| embeddings=embeddings, | |
| documents=texts, | |
| metadatas=metadatas, | |
| ) | |
| logger.info(" Indexed batch %d–%d", i, i + len(batch)) | |
| def semantic_search( | |
| self, | |
| query: str, | |
| k: int = 10, | |
| source_filter: Optional[str] = None, | |
| ) -> List[Dict[str, Any]]: | |
| """Run cosine similarity search. | |
| Returns list of dicts with keys: doc_id, source, score (0-1, higher=better). | |
| """ | |
| if not self._collection or self._collection.count() == 0: | |
| return [] | |
| query_embedding = self._embedding_fn.embed_query(query) | |
| where = {"source": source_filter.upper()} if source_filter else None | |
| results = self._collection.query( | |
| query_embeddings=[query_embedding], | |
| n_results=k, | |
| where=where, | |
| include=["metadatas", "distances"], | |
| ) | |
| hits = [] | |
| if results and results.get("ids") and results["ids"][0]: | |
| for idx, _id in enumerate(results["ids"][0]): | |
| meta = results["metadatas"][0][idx] | |
| # ChromaDB cosine distance: 0 = identical, 2 = opposite | |
| # Convert to similarity: 1 - (distance / 2) | |
| distance = results["distances"][0][idx] | |
| similarity = 1.0 - (distance / 2.0) | |
| hits.append({ | |
| "doc_id": meta.get("doc_id", ""), | |
| "source": meta.get("source", ""), | |
| "score": similarity, | |
| }) | |
| return hits | |