import os import logging import re from collections import Counter from typing import List, Dict, Any, Optional from thefuzz import fuzz from .config import config, SCORING, DATA_DIR from .data_loader import load_internal_research, load_baymard_guidelines, load_nng_articles from .vector_store import VectorStore logger = logging.getLogger("Retriever") # ────────────────────────────────────────────── # Internal-Only Synonym Dictionary # Contains ONLY company-specific abbreviations and names that an LLM cannot infer. # General domain terms (CTR, 이탈, 결제, etc.) are handled by LLM-based query expansion. # ────────────────────────────────────────────── SYNONYMS = { # ── 리서처 이름 (사내 인물) ── "shane": ["shane ahn", "쉐인"], "sonya": ["소냐"], "judy": ["주디"], "욘두": ["yondu"], "라일리": ["riley"], "쟌": ["jan"], # ── 사내 서비스/도구 약어 ── "파트너센터": ["파트너 센터", "협력사 센터", "제휴점 센터", "사장님 사이트"], "bh": ["booking hub", "부킹허브"], "rh": ["resort hub", "리조트허브"], "gc": ["global commerce", "글로벌커머스"], } # Precomputed reverse lookup: synonym_value → canonical_key # Built once at module load instead of iterating SYNONYMS.items() on every call. _SYNONYM_REVERSE: Dict[str, str] = {} for _key, _syns in SYNONYMS.items(): for _syn in _syns: _SYNONYM_REVERSE[_syn.lower()] = _key class Retriever: def __init__(self): logger.info("Initializing Retriever...") # Load Data self.internal_data = load_internal_research() self.baymard_data = load_baymard_guidelines() self.nng_data = load_nng_articles() logger.info(f"Data Loaded: {len(self.internal_data)} Internal, {len(self.baymard_data)} Baymard, {len(self.nng_data)} NN/g") # Initialize VectorStore for semantic search (graceful degradation) self.vector_store: Optional[VectorStore] = None if SCORING.semantic_search_enabled: try: self.vector_store = VectorStore( self.internal_data, self.baymard_data, self.nng_data ) logger.info("VectorStore initialized for semantic search.") except Exception as e: logger.warning("VectorStore init failed, falling back to fuzzy-only: %s", e) self.vector_store = None logger.info("Retriever Ready.") def _expand_keywords(self, keywords: List[str]) -> List[str]: """ Expands keywords using the shared SYNONYMS dictionary. Supports: direct match, reverse match (via precomputed map), and space normalization. Shared across search_internal, search_baymard, search_nng. """ expanded = set(keywords) for kw in keywords: k_lower = kw.lower() # 1. Direct match: keyword is a key in SYNONYMS if k_lower in SYNONYMS: for syn in SYNONYMS[k_lower]: expanded.add(syn) # 2. Reverse match: O(1) lookup via precomputed map if k_lower in _SYNONYM_REVERSE: expanded.add(_SYNONYM_REVERSE[k_lower]) # 3. Space normalization (Korean spacing) if " " in kw: expanded.add(kw.replace(" ", "")) return list(expanded) def _search_external(self, source_label: str, data: List[Dict], keywords: List[str], text_fields: List[str], title_field: str, content_field: str, link_field: str, k: int = 5, min_score: int = 80, date_field: str = None) -> List[Dict]: """ Fuzzy search for external sources (Baymard, NN/g). """ search_keywords = self._expand_keywords(keywords) meta_results = [] for item in data: text = " ".join(item.get(f, '') for f in text_fields) score = self._calculate_fuzzy_score(text, search_keywords) # Title Boost item_title = item.get(title_field, '') for kw in search_keywords: if len(kw) < 2: continue if fuzz.partial_ratio(kw.lower(), item_title.lower()) >= SCORING.fuzzy_title_threshold: score += SCORING.title_boost break if score >= min_score: scored_item = dict(item) scored_item['_score'] = score meta_results.append(scored_item) meta_results.sort(key=lambda x: x.get('_score', 0), reverse=True) top_meta = meta_results[:k] final_results = [] seen = set() def _build_result(item_data: Dict, match_type: str, score: int = 0) -> Dict: result = { 'source': source_label, 'type': match_type, 'id': item_data.get('id'), 'title': item_data.get(title_field), 'content': item_data.get(content_field), 'link': item_data.get(link_field), } if date_field: result['date'] = item_data.get(date_field, '') if score: result['_score'] = score return result # High Confidence for item in top_meta: if item.get('_score', 0) >= SCORING.high_confidence_threshold: rid = item.get('id') if rid in seen: continue final_results.append(_build_result(item, 'Metadata Match (High Confidence)', item.get('_score', 0))) seen.add(rid) # Medium Confidence (remainder) for item in top_meta: rid = item.get('id') if rid in seen: continue final_results.append(_build_result(item, 'Metadata Match (Medium Confidence)', item.get('_score', 0))) seen.add(rid) return final_results def _calculate_fuzzy_score(self, text: str, keywords: List[str]) -> int: if not keywords: return 0 scores = [] for k in keywords: if len(k) < 2: continue score = fuzz.partial_ratio(k.lower(), text.lower()) scores.append(score) if not scores: return 0 # Base score is the best individual match max_score = max(scores) # [Multi-Keyword Bonus] # If multiple unique keywords match significantly, boost the total score. # This helps "Shane UT" rank Shane's UT documents higher than other Shane docs. significant_matches = [s for s in scores if s >= 70] if len(significant_matches) > 1: max_score += (len(significant_matches) - 1) * SCORING.multi_keyword_bonus return min(max_score, SCORING.max_score_cap) def search_internal(self, keywords: List[str], k: int = 100, filters: Dict = None, min_score: int = 80) -> List[Dict]: """Search Internal Research (Fuzzy)""" # 1. Fuzzy Search meta_results = [] # [Synonym Expansion] Use shared method search_keywords = self._expand_keywords(keywords) # [Stop Words] # Generic terms that should not trigger the Title Specific Boost STOP_WORDS = ["리서치", "보고서", "조사", "결과물", "결과", "파일"] for item in self.internal_data: meta = item.get('metadata', {}) detail = item.get('상세 정보', {}) res_id = item.get('id', '') # Exact ID - Force Boost to Top if any(kw.upper() == res_id.upper() for kw in keywords): scored_item = dict(item) scored_item['_score'] = SCORING.exact_id_boost # Massive boost for explicit ID meta_results.insert(0, scored_item) continue text = f"{meta.get('리서치 제목', '')} {meta.get('태그', '')} {meta.get('담당 리서처', '')} {meta.get('방법론', '')} {meta.get('도메인', '')} {detail.get('주요 내용 요약', '')}" score = self._calculate_fuzzy_score(text, search_keywords) # [Automatic Boosting based on Keywords] # Even if 'filters' is not provided, check if keywords match Researcher or Methodology fields item_researcher = meta.get('담당 리서처', '') item_method = meta.get('방법론', '') item_date = meta.get('날짜', '') # 1. Researcher Boost # Check if any keyword matches the researcher name significantly for kw in search_keywords: if len(kw) < 2: continue if fuzz.partial_ratio(kw.lower(), item_researcher.lower()) >= SCORING.fuzzy_researcher_threshold: score += SCORING.researcher_boost # Boost for researcher match break # 2. Methodology Boost for kw in search_keywords: if len(kw) < 2: continue if fuzz.partial_ratio(kw.lower(), item_method.lower()) >= SCORING.fuzzy_method_threshold: score += SCORING.methodology_boost # Boost for methodology match break # 3. Title Specific Boost (NEW) item_title = meta.get('리서치 제목', '') for kw in search_keywords: if len(kw) < 2: continue # [Stop Word Check] Skip boost for generic terms like "리서치" if kw in STOP_WORDS: continue # [Title Match Bonus] # If a keyword matches the TITLE significantly, it's likely very relevant. if fuzz.partial_ratio(kw.lower(), item_title.lower()) >= SCORING.fuzzy_title_threshold: score += SCORING.title_boost # Strong boost for title match break # Methodology Boosting (Legacy Filter Support) if filters: if filters.get('methodology'): target_method = filters['methodology'] if fuzz.partial_ratio(target_method.lower(), item_method.lower()) >= 80: score += SCORING.filter_force_boost # Force boost to top # Researcher Boosting if filters.get('researcher'): target_researcher = filters['researcher'] # Fuzzy match for names (e.g. "Yondu" vs "Yondu Kim") if fuzz.partial_ratio(target_researcher.lower(), item_researcher.lower()) >= SCORING.min_score: score += SCORING.researcher_boost # Significant boost # Date Boosting if filters.get('date'): target_date = filters['date'] # Expected YYYY if target_date in item_date: score += 200 if score >= min_score: # Use dynamic threshold (Default 80) scored_item = dict(item) scored_item['_score'] = score meta_results.append(scored_item) meta_results.sort(key=lambda x: x.get('_score', 0), reverse=True) if k is None: k = 100 k = int(k) top_meta = meta_results[:k] final_results = [] seen = set() def _build_internal_result(item, match_type): meta = item.get('metadata', {}) detail = item.get('상세 정보', {}) related = detail.get('관련 자료', []) links_str = "" if related: links_str = "\nRelated Materials:\n" + "\n".join([f"- {l.get('제목')}: {l.get('url')}" for l in related]) return { 'source': 'INTERNAL', 'type': match_type, 'id': item.get('id'), 'title': meta.get('리서치 제목'), 'date': meta.get('날짜', 'N/A'), 'researcher': meta.get('담당 리서처', 'N/A'), 'methodology': meta.get('방법론', 'N/A'), 'domain': meta.get('도메인', 'N/A'), 'content': detail.get('주요 내용 요약'), 'link': f"{meta.get('결과 링크')}{links_str}", '_score': item.get('_score', 0) } # High Confidence for item in top_meta: if item.get('_score', 0) >= SCORING.high_confidence_threshold: rid = item.get('id') if rid in seen: continue final_results.append(_build_internal_result(item, 'Metadata Match (High Confidence)')) seen.add(rid) # Medium Confidence (remainder) for item in top_meta: rid = item.get('id') if rid in seen: continue final_results.append(_build_internal_result(item, 'Metadata Match (Medium Confidence)')) seen.add(rid) # 3. Global Sort by Score (Primary) and Date (Secondary) def sort_key(item): # Primary: Score (DESC) score = item.get('_score', 0) # Secondary: Date (DESC) d_str = item.get('date', '') clean_d = re.sub(r'[^0-9]', '', str(d_str)) date_val = 0 try: if clean_d: date_val = int(clean_d) except ValueError: pass return (score, date_val) final_results.sort(key=sort_key, reverse=True) return final_results def search_baymard(self, keywords: List[str], k=5, min_score: int = 80) -> List[Dict]: """Search Baymard Guidelines""" return self._search_external( source_label='BAYMARD', data=self.baymard_data, keywords=keywords, text_fields=['title', 'description', 'issue'], title_field='title', content_field='advice', link_field='url', k=k, min_score=min_score, ) def search_nng(self, keywords: List[str], k=5, min_score: int = 80) -> List[Dict]: """Search NN/g Articles""" return self._search_external( source_label='NNG', data=self.nng_data, keywords=keywords, text_fields=['title', 'description'], title_field='title', content_field='description', link_field='url', k=k, min_score=min_score, date_field='date', ) # ────────────────────────────────────────────── # Hybrid Search (Fuzzy + Semantic via RRF) # ────────────────────────────────────────────── def hybrid_search( self, keywords: List[str], query: str, source: Optional[str] = None, k: int = 10, min_score: int = 60, ) -> List[Dict]: """Combine fuzzy and semantic search results using Reciprocal Rank Fusion. Args: keywords: Keywords for fuzzy search. query: Natural language query for semantic search. source: Optional source filter ("INTERNAL", "BAYMARD", "NNG"). k: Max results to return. min_score: Minimum fuzzy score threshold. Returns: Merged results sorted by RRF score, each dict includes 'match_type'. """ rrf_k = SCORING.rrf_k sources_to_search = [source.upper()] if source else ["INTERNAL", "BAYMARD", "NNG"] # 1. Collect fuzzy results fuzzy_results = [] for src in sources_to_search: if src == "INTERNAL": fuzzy_results.extend(self.search_internal(keywords, k=k, min_score=min_score)) elif src == "BAYMARD": fuzzy_results.extend(self.search_baymard(keywords, k=k, min_score=min_score)) elif src == "NNG": fuzzy_results.extend(self.search_nng(keywords, k=k, min_score=min_score)) # 2. Collect semantic results semantic_hits = [] if self.vector_store and query: semantic_hits = self.vector_store.semantic_search( query=query, k=k, source_filter=source ) # If no semantic results, return fuzzy results as-is if not semantic_hits: for r in fuzzy_results: r["match_type"] = "Fuzzy Match" return fuzzy_results[:k] # 3. Build rank maps (doc_id → rank, 0-indexed) fuzzy_rank = {} for rank, item in enumerate(fuzzy_results): doc_id = str(item.get("id", "")) src = item.get("source", "") key = f"{src}:{doc_id}" if key not in fuzzy_rank: fuzzy_rank[key] = rank semantic_rank = {} for rank, hit in enumerate(semantic_hits): key = f"{hit['source']}:{hit['doc_id']}" if key not in semantic_rank: semantic_rank[key] = rank # 4. Compute RRF scores all_keys = set(fuzzy_rank.keys()) | set(semantic_rank.keys()) rrf_scores = {} for key in all_keys: score = 0.0 if key in fuzzy_rank: score += 1.0 / (rrf_k + fuzzy_rank[key]) if key in semantic_rank: score += 1.0 / (rrf_k + semantic_rank[key]) rrf_scores[key] = score # 5. Build result list # Map fuzzy results by key for quick lookup fuzzy_map = {} for item in fuzzy_results: key = f"{item.get('source', '')}:{item.get('id', '')}" if key not in fuzzy_map: fuzzy_map[key] = item sorted_keys = sorted(rrf_scores.keys(), key=lambda k: rrf_scores[k], reverse=True) merged = [] for key in sorted_keys[:k]: in_fuzzy = key in fuzzy_rank in_semantic = key in semantic_rank if in_fuzzy and in_semantic: match_type = "Hybrid Match" elif in_fuzzy: match_type = "Fuzzy Match" else: match_type = "Semantic Match" if key in fuzzy_map: result = dict(fuzzy_map[key]) result["match_type"] = match_type merged.append(result) else: # Semantic-only hit: build result from document data result = self._build_result_from_semantic(key) if result: result["match_type"] = match_type merged.append(result) return merged def _build_result_from_semantic(self, key: str) -> Optional[Dict]: """Build a search result dict for a semantic-only hit.""" parts = key.split(":", 1) if len(parts) != 2: return None source, doc_id = parts doc = self.get_document_by_id(doc_id, source) if not doc: return None if source == "INTERNAL": return { "source": "INTERNAL", "type": "Semantic Match", "id": doc_id, "title": doc.get("title", ""), "date": doc.get("date", ""), "researcher": doc.get("researcher", ""), "methodology": doc.get("methodology", ""), "domain": doc.get("domain", ""), "content": doc.get("summary", ""), "link": doc.get("link", ""), } elif source == "BAYMARD": return { "source": "BAYMARD", "type": "Semantic Match", "id": doc_id, "title": doc.get("title", ""), "content": doc.get("advice", "") or doc.get("full_content", ""), "link": doc.get("link", ""), } elif source == "NNG": return { "source": "NNG", "type": "Semantic Match", "id": doc_id, "title": doc.get("title", ""), "content": doc.get("description", ""), "link": doc.get("link", ""), "date": doc.get("date", ""), } return None def get_specific_item(self, query: str) -> Dict[str, Any]: """ Finds a specific item (Internal/Baymard/NNG) best matching the query (Title/ID/Link). Returns Dict with 'content' (FULL), 'title', 'source', etc. or None if not found. """ # 1. Normalize query query_norm = query.strip().lower() best_match = None best_score = 0 # Helper to check match def check_match(item, source_type): nonlocal best_match, best_score # Fields to check rid = str(item.get('id', '')) # Handle different data structures if source_type == 'INTERNAL': meta = item.get('metadata', {}) title = meta.get('리서치 제목', '') link = meta.get('결과 링크', '') content = item.get('상세 정보', {}).get('주요 내용 요약', '') # Will enrich later elif source_type == 'BAYMARD': title = item.get('title', '') link = item.get('url', '') content = item.get('advice', '') elif source_type == 'NNG': title = item.get('title', '') link = item.get('url', '') content = item.get('description', '') # Calculate Score # Exact ID Match if rid.lower() in query_norm: score = 100 # URL Match elif query_norm in link.lower() and len(query_norm) > 10: score = 100 # Title Fuzzy Match else: score = fuzz.ratio(query_norm, title.lower()) # Partial match for long titles if query is significant part if len(query_norm) > 5: partial = fuzz.partial_ratio(query_norm, title.lower()) if partial > score: score = partial if score > 85 and score > best_score: best_score = score best_match = { "source": source_type, "id": rid, "title": title, "link": link, "content": content, # Placeholder, will fetch full doc if needed "original_item": item } # Scan Data sources for item in self.internal_data: check_match(item, 'INTERNAL') for item in self.baymard_data: check_match(item, 'BAYMARD') for item in self.nng_data: check_match(item, 'NNG') if best_match: logger.info(f"Specific Item Match: {best_match['title']} ({best_match['source']}) - Score: {best_score}") return best_match return None # ────────────────────────────────────────────── # Agentic Retriever Helper Methods # ────────────────────────────────────────────── def _read_md_file(self, file_path: str) -> str: """Read a markdown file from DATA_DIR. Returns content or empty string.""" try: full_path = os.path.join(str(DATA_DIR), file_path.lstrip('/')) if os.path.exists(full_path): with open(full_path, 'r', encoding='utf-8') as f: return f.read() except Exception as e: logger.warning(f"Failed to read MD file {file_path}: {e}") return "" def get_document_by_id(self, doc_id: str, source: str) -> Optional[Dict[str, Any]]: """Returns full document content by ID and source. Used by agentic retriever tools.""" source = source.upper() if source == "INTERNAL": item = next((d for d in self.internal_data if d.get('id') == doc_id), None) if not item: return None meta = item.get('metadata', {}) detail = item.get('상세 정보', {}) # Read full MD content if available file_path = meta.get('file_path', '') md_content = self._read_md_file(file_path) if file_path and file_path != "파일 없음" else "" related = detail.get('관련 자료', []) links_str = "" if related: links_str = "\n관련 자료:\n" + "\n".join([f"- {l.get('제목')}: {l.get('url')}" for l in related]) return { 'id': doc_id, 'source': 'INTERNAL', 'title': meta.get('리서치 제목', ''), 'date': meta.get('날짜', ''), 'researcher': meta.get('담당 리서처', ''), 'methodology': meta.get('방법론', ''), 'domain': meta.get('도메인', ''), 'tags': meta.get('태그', ''), 'summary': detail.get('주요 내용 요약', ''), 'full_content': md_content or detail.get('주요 내용 요약', ''), 'link': meta.get('결과 링크', '') + links_str, } elif source == "BAYMARD": item = next((d for d in self.baymard_data if str(d.get('id')) == str(doc_id)), None) if not item: return None return { 'id': str(item.get('id')), 'source': 'BAYMARD', 'title': item.get('title', ''), 'topic': item.get('topic', ''), 'issue': item.get('issue', ''), 'advice': item.get('advice', ''), 'description': item.get('description', ''), 'full_content': f"Issue:\n{item.get('issue', '')}\n\nAdvice:\n{item.get('advice', '')}\n\nDescription:\n{item.get('description', '')}", 'link': item.get('url', ''), } elif source == "NNG": item = next((d for d in self.nng_data if d.get('id') == doc_id), None) if not item: return None contents_list = item.get('contents', []) contents_str = "\n".join(f"- {c}" for c in contents_list) if isinstance(contents_list, list) else str(contents_list) return { 'id': doc_id, 'source': 'NNG', 'title': item.get('title', ''), 'date': item.get('date', ''), 'authors': item.get('authors', []), 'topics': item.get('topics', []), 'description': item.get('description', ''), 'contents_outline': contents_str, 'full_content': f"Description:\n{item.get('description', '')}\n\nContents:\n{contents_str}", 'link': item.get('url', ''), } return None def list_topics(self, source: str) -> List[Dict[str, Any]]: """Returns list of topics with document counts for a given source.""" source = source.upper() if source == "BAYMARD": counter = Counter(item.get('topic', 'Unknown') for item in self.baymard_data) return [{'topic': topic, 'count': count} for topic, count in counter.most_common()] elif source == "NNG": counter: Counter = Counter() for item in self.nng_data: for t in item.get('topics', []): counter[t] += 1 return [{'topic': topic, 'count': count} for topic, count in counter.most_common()] elif source == "INTERNAL": counter = Counter(item.get('metadata', {}).get('도메인', 'Unknown') for item in self.internal_data) return [{'topic': topic, 'count': count} for topic, count in counter.most_common()] return [] def list_docs_by_topic(self, source: str, topic: str) -> List[Dict[str, Any]]: """Returns document ID/title/preview for a given source and topic.""" source = source.upper() results = [] if source == "BAYMARD": for item in self.baymard_data: if item.get('topic', '').lower() == topic.lower(): results.append({ 'id': str(item.get('id')), 'title': item.get('title', ''), 'preview': (item.get('description', '') or item.get('issue', ''))[:120], }) elif source == "NNG": for item in self.nng_data: if topic.lower() in [t.lower() for t in item.get('topics', [])]: results.append({ 'id': item.get('id', ''), 'title': item.get('title', ''), 'preview': (item.get('description', ''))[:120], }) elif source == "INTERNAL": for item in self.internal_data: if item.get('metadata', {}).get('도메인', '').lower() == topic.lower(): meta = item.get('metadata', {}) results.append({ 'id': item.get('id', ''), 'title': meta.get('리서치 제목', ''), 'preview': (item.get('상세 정보', {}).get('주요 내용 요약', ''))[:120], }) return results # ────────────────────────────────────────────── # Aggregation # ────────────────────────────────────────────── _INTERNAL_GROUP_FIELDS = { "year": lambda item: item.get('metadata', {}).get('날짜', '')[:4] or 'Unknown', "domain": lambda item: item.get('metadata', {}).get('도메인', '') or 'Unknown', "methodology": lambda item: item.get('metadata', {}).get('방법론', '') or 'Unknown', "researcher": lambda item: item.get('metadata', {}).get('담당 리서처', '') or 'Unknown', "tag": lambda item: item.get('metadata', {}).get('태그', '') or 'Unknown', } _BAYMARD_GROUP_FIELDS = { "topic": lambda item: [item.get('topic', 'Unknown')], "theme": lambda item: [item.get('theme', 'Unknown')], "platform": lambda item: item.get('platforms', []) or ['Unknown'], "importance": lambda item: [str(item.get('importance', 'Unknown'))], } _NNG_GROUP_FIELDS = { "topic": lambda item: item.get('topics', []) or ['Unknown'], "author": lambda item: item.get('authors', []) or ['Unknown'], "year": lambda item: [Retriever._extract_year_from_text(item.get('date', ''))], } @staticmethod def _extract_year_from_text(date_text: str) -> str: """Extract a 4-digit year from free-form date text like 'June 25, 2023'.""" if not date_text: return 'Unknown' m = re.search(r'\b(19|20)\d{2}\b', date_text) return m.group(0) if m else 'Unknown' def aggregate(self, source: str, group_by: str, filter_field: str = None, filter_value: str = None) -> Dict: """Aggregate document counts by a metadata field, with optional filtering. Args: source: "INTERNAL", "BAYMARD", or "NNG". group_by: Field name to group by (source-specific). filter_field: Optional field to filter on before aggregating. filter_value: Value for filter (case-insensitive partial match). Returns: Dict with source, group_by, total_docs, counts (list of (value, count) tuples), and optional filter info. """ source = source.upper() # ALL: aggregate across all sources, grouped by source name if source == "ALL": counts = [ ("INTERNAL", len(self.internal_data)), ("BAYMARD", len(self.baymard_data)), ("NNG", len(self.nng_data)), ] total = sum(c for _, c in counts) return { "source": "ALL", "group_by": "source", "total_docs": total, "counts": counts, "filter": None, } # Select data and field maps if source == "INTERNAL": data = self.internal_data group_map = self._INTERNAL_GROUP_FIELDS filter_map = self._INTERNAL_GROUP_FIELDS elif source == "BAYMARD": data = self.baymard_data group_map = self._BAYMARD_GROUP_FIELDS filter_map = self._BAYMARD_GROUP_FIELDS elif source == "NNG": data = self.nng_data group_map = self._NNG_GROUP_FIELDS filter_map = self._NNG_GROUP_FIELDS else: return {"error": f"Unknown source: {source}. Use INTERNAL, BAYMARD, NNG, or ALL."} if group_by not in group_map: valid = ", ".join(group_map.keys()) return {"error": f"Invalid group_by '{group_by}' for {source}. Valid: {valid}"} # Validate filter_field if filter_field and filter_field not in filter_map: valid = ", ".join(filter_map.keys()) return {"error": f"Invalid filter_field '{filter_field}' for {source}. Valid: {valid}"} group_fn = group_map[group_by] # Apply optional filter filtered_data = data if filter_field and filter_value: filter_fn = filter_map[filter_field] fv_lower = filter_value.lower() filtered_data = [] for item in data: values = filter_fn(item) if isinstance(values, str): values = [values] if any(fv_lower in v.lower() for v in values): filtered_data.append(item) # Count counter: Counter = Counter() for item in filtered_data: values = group_fn(item) if isinstance(values, str): values = [values] for v in values: counter[v] += 1 return { "source": source, "group_by": group_by, "total_docs": len(filtered_data), "counts": counter.most_common(), "filter": {"field": filter_field, "value": filter_value} if filter_field else None, } def filter_by_metadata(self, source: str, field: str, value: str, max_results: int = 20) -> List[Dict]: """Filter documents by metadata field and return matching documents with previews. Args: source: "INTERNAL", "BAYMARD", or "NNG". field: Metadata field to filter on (same as aggregate group_by fields). value: Filter value (case-insensitive partial match). max_results: Maximum documents to return. Returns: List of matching documents with id, title, preview, and metadata. """ source = source.upper() field_maps = { "INTERNAL": (self.internal_data, self._INTERNAL_GROUP_FIELDS), "BAYMARD": (self.baymard_data, self._BAYMARD_GROUP_FIELDS), "NNG": (self.nng_data, self._NNG_GROUP_FIELDS), } if source not in field_maps: return [] data, group_map = field_maps[source] if field not in group_map: return [] filter_fn = group_map[field] val_lower = value.lower() results = [] for item in data: values = filter_fn(item) if isinstance(values, str): values = [values] if not any(val_lower in v.lower() for v in values): continue if source == "INTERNAL": meta = item.get('metadata', {}) detail = item.get('상세 정보', {}) results.append({ 'id': item.get('id', ''), 'title': meta.get('리서치 제목', ''), 'source': 'INTERNAL', 'date': meta.get('날짜', ''), 'domain': meta.get('도메인', ''), 'methodology': meta.get('방법론', ''), 'researcher': meta.get('담당 리서처', ''), 'tags': meta.get('태그', ''), 'link': meta.get('결과 링크', ''), 'preview': (detail.get('주요 내용 요약', ''))[:250], }) elif source == "BAYMARD": results.append({ 'id': item.get('id', ''), 'title': item.get('title', ''), 'source': 'BAYMARD', 'topic': item.get('topic', ''), 'importance': item.get('importance', ''), 'link': item.get('url', ''), 'preview': (item.get('description', ''))[:250], }) elif source == "NNG": results.append({ 'id': item.get('id', ''), 'title': item.get('title', ''), 'source': 'NNG', 'authors': item.get('authors', []), 'topics': item.get('topics', []), 'link': item.get('url', ''), 'preview': (item.get('description', ''))[:250], }) if len(results) >= max_results: break return results