debug-log
Claude Opus 4.6
feat: Insight Repository — GitLab 초기화 + HF 자동 배포
501bc8c
Raw History Blame Contribute Delete
8.37 kB
"""ChromaDB-based vector store for semantic search.
Uses Google text-embedding-004 to embed documents from all three sources
(INTERNAL, BAYMARD, NNG) and provides cosine similarity search.
Gracefully degrades: if initialization fails, callers fall back to fuzzy-only search.
"""
import hashlib
import json
import logging
from pathlib import Path
from typing import List, Dict, Any, Optional
from .config import config, SCORING, PROJECT_ROOT
logger = logging.getLogger("VectorStore")
VECTOR_STORE_DIR = PROJECT_ROOT / ".vector_store"
HASH_FILE = VECTOR_STORE_DIR / "data_hash.json"
COLLECTION_NAME = "ux_research_docs"
def _build_internal_text(item: Dict[str, Any], max_len: int) -> str:
"""Build embedding text for an INTERNAL research document."""
meta = item.get("metadata", {})
detail = item.get("상세 정보", {})
parts = [
meta.get("리서치 제목", ""),
meta.get("태그", ""),
meta.get("도메인", ""),
meta.get("방법론", ""),
detail.get("주요 내용 요약", ""),
]
return " ".join(p for p in parts if p)[:max_len]
def _build_baymard_text(item: Dict[str, Any], max_len: int) -> str:
"""Build embedding text for a BAYMARD guideline."""
parts = [
item.get("title", ""),
item.get("topic", ""),
item.get("issue", ""),
item.get("advice", ""),
]
return " ".join(p for p in parts if p)[:max_len]
def _build_nng_text(item: Dict[str, Any], max_len: int) -> str:
"""Build embedding text for a NNG article."""
contents = item.get("contents", [])
contents_str = " ".join(contents) if isinstance(contents, list) else str(contents)
parts = [
item.get("title", ""),
item.get("description", ""),
" ".join(item.get("topics", [])),
contents_str,
]
return " ".join(p for p in parts if p)[:max_len]
class VectorStore:
"""ChromaDB vector store with Google Generative AI embeddings."""
def __init__(
self,
internal_data: List[Dict],
baymard_data: List[Dict],
nng_data: List[Dict],
):
self.internal_data = internal_data
self.baymard_data = baymard_data
self.nng_data = nng_data
self.max_len = SCORING.max_embedding_text_length
self.batch_size = SCORING.embedding_batch_size
self._collection = None
self._initialize()
def _compute_data_hash(self) -> str:
"""Compute a SHA-256 hash from document counts and boundary IDs."""
parts = []
for label, data, id_fn in [
("internal", self.internal_data, lambda d: d.get("id", "")),
("baymard", self.baymard_data, lambda d: str(d.get("id", ""))),
("nng", self.nng_data, lambda d: d.get("id", "")),
]:
count = len(data)
first_id = id_fn(data[0]) if data else ""
last_id = id_fn(data[-1]) if data else ""
parts.append(f"{label}:{count}:{first_id}:{last_id}")
raw = "|".join(parts)
return hashlib.sha256(raw.encode()).hexdigest()
def _initialize(self):
"""Load existing index or rebuild if data hash changed."""
import chromadb
from langchain_google_genai import GoogleGenerativeAIEmbeddings
self._embedding_fn = GoogleGenerativeAIEmbeddings(
model="models/text-embedding-004",
google_api_key=config.google_api_key,
)
VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True)
self._client = chromadb.PersistentClient(path=str(VECTOR_STORE_DIR))
current_hash = self._compute_data_hash()
# Check if existing index matches current data
stored_hash = None
if HASH_FILE.exists():
try:
stored_hash = json.loads(HASH_FILE.read_text()).get("hash")
except Exception:
pass
if stored_hash == current_hash:
# Reuse existing collection
try:
self._collection = self._client.get_collection(name=COLLECTION_NAME)
if self._collection.count() > 0:
logger.info(
"Loaded existing vector index (%d docs, hash=%s…)",
self._collection.count(),
current_hash[:8],
)
return
except Exception:
pass
# Rebuild
logger.info("Building vector index (hash=%s…)…", current_hash[:8])
self._build_index()
# Save hash
HASH_FILE.write_text(json.dumps({"hash": current_hash}))
logger.info("Vector index built and saved.")
def _build_index(self):
"""Embed all documents and store in ChromaDB."""
# Delete old collection if exists
try:
self._client.delete_collection(name=COLLECTION_NAME)
except Exception:
pass
self._collection = self._client.create_collection(
name=COLLECTION_NAME,
metadata={"hnsw:space": "cosine"},
)
all_docs = [] # (id, text, metadata)
for item in self.internal_data:
doc_id = item.get("id", "")
if not doc_id:
continue
text = _build_internal_text(item, self.max_len)
if text.strip():
all_docs.append((
f"INTERNAL:{doc_id}",
text,
{"source": "INTERNAL", "doc_id": doc_id},
))
for item in self.baymard_data:
doc_id = str(item.get("id", ""))
if not doc_id:
continue
text = _build_baymard_text(item, self.max_len)
if text.strip():
all_docs.append((
f"BAYMARD:{doc_id}",
text,
{"source": "BAYMARD", "doc_id": doc_id},
))
for item in self.nng_data:
doc_id = item.get("id", "")
if not doc_id:
continue
text = _build_nng_text(item, self.max_len)
if text.strip():
all_docs.append((
f"NNG:{doc_id}",
text,
{"source": "NNG", "doc_id": doc_id},
))
logger.info("Embedding %d documents…", len(all_docs))
# Batch embed and upsert
for i in range(0, len(all_docs), self.batch_size):
batch = all_docs[i : i + self.batch_size]
ids = [d[0] for d in batch]
texts = [d[1] for d in batch]
metadatas = [d[2] for d in batch]
embeddings = self._embedding_fn.embed_documents(texts)
self._collection.add(
ids=ids,
embeddings=embeddings,
documents=texts,
metadatas=metadatas,
)
logger.info(" Indexed batch %d–%d", i, i + len(batch))
def semantic_search(
self,
query: str,
k: int = 10,
source_filter: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Run cosine similarity search.
Returns list of dicts with keys: doc_id, source, score (0-1, higher=better).
"""
if not self._collection or self._collection.count() == 0:
return []
query_embedding = self._embedding_fn.embed_query(query)
where = {"source": source_filter.upper()} if source_filter else None
results = self._collection.query(
query_embeddings=[query_embedding],
n_results=k,
where=where,
include=["metadatas", "distances"],
)
hits = []
if results and results.get("ids") and results["ids"][0]:
for idx, _id in enumerate(results["ids"][0]):
meta = results["metadatas"][0][idx]
# ChromaDB cosine distance: 0 = identical, 2 = opposite
# Convert to similarity: 1 - (distance / 2)
distance = results["distances"][0][idx]
similarity = 1.0 - (distance / 2.0)
hits.append({
"doc_id": meta.get("doc_id", ""),
"source": meta.get("source", ""),
"score": similarity,
})
return hits