"""ChromaDB-based vector store for semantic search. Uses Google text-embedding-004 to embed documents from all three sources (INTERNAL, BAYMARD, NNG) and provides cosine similarity search. Gracefully degrades: if initialization fails, callers fall back to fuzzy-only search. """ import hashlib import json import logging from pathlib import Path from typing import List, Dict, Any, Optional from .config import config, SCORING, PROJECT_ROOT logger = logging.getLogger("VectorStore") VECTOR_STORE_DIR = PROJECT_ROOT / ".vector_store" HASH_FILE = VECTOR_STORE_DIR / "data_hash.json" COLLECTION_NAME = "ux_research_docs" def _build_internal_text(item: Dict[str, Any], max_len: int) -> str: """Build embedding text for an INTERNAL research document.""" meta = item.get("metadata", {}) detail = item.get("상세 정보", {}) parts = [ meta.get("리서치 제목", ""), meta.get("태그", ""), meta.get("도메인", ""), meta.get("방법론", ""), detail.get("주요 내용 요약", ""), ] return " ".join(p for p in parts if p)[:max_len] def _build_baymard_text(item: Dict[str, Any], max_len: int) -> str: """Build embedding text for a BAYMARD guideline.""" parts = [ item.get("title", ""), item.get("topic", ""), item.get("issue", ""), item.get("advice", ""), ] return " ".join(p for p in parts if p)[:max_len] def _build_nng_text(item: Dict[str, Any], max_len: int) -> str: """Build embedding text for a NNG article.""" contents = item.get("contents", []) contents_str = " ".join(contents) if isinstance(contents, list) else str(contents) parts = [ item.get("title", ""), item.get("description", ""), " ".join(item.get("topics", [])), contents_str, ] return " ".join(p for p in parts if p)[:max_len] class VectorStore: """ChromaDB vector store with Google Generative AI embeddings.""" def __init__( self, internal_data: List[Dict], baymard_data: List[Dict], nng_data: List[Dict], ): self.internal_data = internal_data self.baymard_data = baymard_data self.nng_data = nng_data self.max_len = SCORING.max_embedding_text_length self.batch_size = SCORING.embedding_batch_size self._collection = None self._initialize() def _compute_data_hash(self) -> str: """Compute a SHA-256 hash from document counts and boundary IDs.""" parts = [] for label, data, id_fn in [ ("internal", self.internal_data, lambda d: d.get("id", "")), ("baymard", self.baymard_data, lambda d: str(d.get("id", ""))), ("nng", self.nng_data, lambda d: d.get("id", "")), ]: count = len(data) first_id = id_fn(data[0]) if data else "" last_id = id_fn(data[-1]) if data else "" parts.append(f"{label}:{count}:{first_id}:{last_id}") raw = "|".join(parts) return hashlib.sha256(raw.encode()).hexdigest() def _initialize(self): """Load existing index or rebuild if data hash changed.""" import chromadb from langchain_google_genai import GoogleGenerativeAIEmbeddings self._embedding_fn = GoogleGenerativeAIEmbeddings( model="models/text-embedding-004", google_api_key=config.google_api_key, ) VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True) self._client = chromadb.PersistentClient(path=str(VECTOR_STORE_DIR)) current_hash = self._compute_data_hash() # Check if existing index matches current data stored_hash = None if HASH_FILE.exists(): try: stored_hash = json.loads(HASH_FILE.read_text()).get("hash") except Exception: pass if stored_hash == current_hash: # Reuse existing collection try: self._collection = self._client.get_collection(name=COLLECTION_NAME) if self._collection.count() > 0: logger.info( "Loaded existing vector index (%d docs, hash=%s…)", self._collection.count(), current_hash[:8], ) return except Exception: pass # Rebuild logger.info("Building vector index (hash=%s…)…", current_hash[:8]) self._build_index() # Save hash HASH_FILE.write_text(json.dumps({"hash": current_hash})) logger.info("Vector index built and saved.") def _build_index(self): """Embed all documents and store in ChromaDB.""" # Delete old collection if exists try: self._client.delete_collection(name=COLLECTION_NAME) except Exception: pass self._collection = self._client.create_collection( name=COLLECTION_NAME, metadata={"hnsw:space": "cosine"}, ) all_docs = [] # (id, text, metadata) for item in self.internal_data: doc_id = item.get("id", "") if not doc_id: continue text = _build_internal_text(item, self.max_len) if text.strip(): all_docs.append(( f"INTERNAL:{doc_id}", text, {"source": "INTERNAL", "doc_id": doc_id}, )) for item in self.baymard_data: doc_id = str(item.get("id", "")) if not doc_id: continue text = _build_baymard_text(item, self.max_len) if text.strip(): all_docs.append(( f"BAYMARD:{doc_id}", text, {"source": "BAYMARD", "doc_id": doc_id}, )) for item in self.nng_data: doc_id = item.get("id", "") if not doc_id: continue text = _build_nng_text(item, self.max_len) if text.strip(): all_docs.append(( f"NNG:{doc_id}", text, {"source": "NNG", "doc_id": doc_id}, )) logger.info("Embedding %d documents…", len(all_docs)) # Batch embed and upsert for i in range(0, len(all_docs), self.batch_size): batch = all_docs[i : i + self.batch_size] ids = [d[0] for d in batch] texts = [d[1] for d in batch] metadatas = [d[2] for d in batch] embeddings = self._embedding_fn.embed_documents(texts) self._collection.add( ids=ids, embeddings=embeddings, documents=texts, metadatas=metadatas, ) logger.info(" Indexed batch %d–%d", i, i + len(batch)) def semantic_search( self, query: str, k: int = 10, source_filter: Optional[str] = None, ) -> List[Dict[str, Any]]: """Run cosine similarity search. Returns list of dicts with keys: doc_id, source, score (0-1, higher=better). """ if not self._collection or self._collection.count() == 0: return [] query_embedding = self._embedding_fn.embed_query(query) where = {"source": source_filter.upper()} if source_filter else None results = self._collection.query( query_embeddings=[query_embedding], n_results=k, where=where, include=["metadatas", "distances"], ) hits = [] if results and results.get("ids") and results["ids"][0]: for idx, _id in enumerate(results["ids"][0]): meta = results["metadatas"][0][idx] # ChromaDB cosine distance: 0 = identical, 2 = opposite # Convert to similarity: 1 - (distance / 2) distance = results["distances"][0][idx] similarity = 1.0 - (distance / 2.0) hits.append({ "doc_id": meta.get("doc_id", ""), "source": meta.get("source", ""), "score": similarity, }) return hits