Download src/engine/retriever.py from YGCXLab/ux-research-agent-slackbot: direct link, hf CLI and curl.
- Browser
- Download file 38.1 kB
-
https://huggingface.co/spaces/YGCXLab/ux-research-agent-slackbot/resolve/main/src/engine/retriever.py
- Command line
-
hf download hf://spaces/YGCXLab/ux-research-agent-slackbot/src/engine/retriever.py
-
curl -L -o retriever.py https://huggingface.co/spaces/YGCXLab/ux-research-agent-slackbot/resolve/main/src/engine/retriever.py
38.1 kB
| import os | |
| import logging | |
| import re | |
| from collections import Counter | |
| from typing import List, Dict, Any, Optional | |
| from thefuzz import fuzz | |
| from .config import config, SCORING, DATA_DIR | |
| from .data_loader import load_internal_research, load_baymard_guidelines, load_nng_articles | |
| from .vector_store import VectorStore | |
| logger = logging.getLogger("Retriever") | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Internal-Only Synonym Dictionary | |
| # Contains ONLY company-specific abbreviations and names that an LLM cannot infer. | |
| # General domain terms (CTR, μ΄ν, κ²°μ , etc.) are handled by LLM-based query expansion. | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| SYNONYMS = { | |
| # ββ 리μμ² μ΄λ¦ (μ¬λ΄ μΈλ¬Ό) ββ | |
| "shane": ["shane ahn", "μμΈ"], | |
| "sonya": ["μλ"], | |
| "judy": ["μ£Όλ"], | |
| "μλ": ["yondu"], | |
| "λΌμΌλ¦¬": ["riley"], | |
| "μ": ["jan"], | |
| # ββ μ¬λ΄ μλΉμ€/λꡬ μ½μ΄ ββ | |
| "ννΈλμΌν°": ["ννΈλ μΌν°", "νλ ₯μ¬ μΌν°", "μ ν΄μ μΌν°", "μ¬μ₯λ μ¬μ΄νΈ"], | |
| "bh": ["booking hub", "λΆνΉνλΈ"], | |
| "rh": ["resort hub", "리쑰νΈνλΈ"], | |
| "gc": ["global commerce", "κΈλ‘λ²μ»€λ¨Έμ€"], | |
| } | |
| # Precomputed reverse lookup: synonym_value β canonical_key | |
| # Built once at module load instead of iterating SYNONYMS.items() on every call. | |
| _SYNONYM_REVERSE: Dict[str, str] = {} | |
| for _key, _syns in SYNONYMS.items(): | |
| for _syn in _syns: | |
| _SYNONYM_REVERSE[_syn.lower()] = _key | |
| class Retriever: | |
| def __init__(self): | |
| logger.info("Initializing Retriever...") | |
| # Load Data | |
| self.internal_data = load_internal_research() | |
| self.baymard_data = load_baymard_guidelines() | |
| self.nng_data = load_nng_articles() | |
| logger.info(f"Data Loaded: {len(self.internal_data)} Internal, {len(self.baymard_data)} Baymard, {len(self.nng_data)} NN/g") | |
| # Initialize VectorStore for semantic search (graceful degradation) | |
| self.vector_store: Optional[VectorStore] = None | |
| if SCORING.semantic_search_enabled: | |
| try: | |
| self.vector_store = VectorStore( | |
| self.internal_data, self.baymard_data, self.nng_data | |
| ) | |
| logger.info("VectorStore initialized for semantic search.") | |
| except Exception as e: | |
| logger.warning("VectorStore init failed, falling back to fuzzy-only: %s", e) | |
| self.vector_store = None | |
| logger.info("Retriever Ready.") | |
| def _expand_keywords(self, keywords: List[str]) -> List[str]: | |
| """ | |
| Expands keywords using the shared SYNONYMS dictionary. | |
| Supports: direct match, reverse match (via precomputed map), and space normalization. | |
| Shared across search_internal, search_baymard, search_nng. | |
| """ | |
| expanded = set(keywords) | |
| for kw in keywords: | |
| k_lower = kw.lower() | |
| # 1. Direct match: keyword is a key in SYNONYMS | |
| if k_lower in SYNONYMS: | |
| for syn in SYNONYMS[k_lower]: | |
| expanded.add(syn) | |
| # 2. Reverse match: O(1) lookup via precomputed map | |
| if k_lower in _SYNONYM_REVERSE: | |
| expanded.add(_SYNONYM_REVERSE[k_lower]) | |
| # 3. Space normalization (Korean spacing) | |
| if " " in kw: | |
| expanded.add(kw.replace(" ", "")) | |
| return list(expanded) | |
| def _search_external(self, source_label: str, data: List[Dict], | |
| keywords: List[str], text_fields: List[str], | |
| title_field: str, content_field: str, link_field: str, | |
| k: int = 5, min_score: int = 80, | |
| date_field: str = None) -> List[Dict]: | |
| """ | |
| Fuzzy search for external sources (Baymard, NN/g). | |
| """ | |
| search_keywords = self._expand_keywords(keywords) | |
| meta_results = [] | |
| for item in data: | |
| text = " ".join(item.get(f, '') for f in text_fields) | |
| score = self._calculate_fuzzy_score(text, search_keywords) | |
| # Title Boost | |
| item_title = item.get(title_field, '') | |
| for kw in search_keywords: | |
| if len(kw) < 2: continue | |
| if fuzz.partial_ratio(kw.lower(), item_title.lower()) >= SCORING.fuzzy_title_threshold: | |
| score += SCORING.title_boost | |
| break | |
| if score >= min_score: | |
| scored_item = dict(item) | |
| scored_item['_score'] = score | |
| meta_results.append(scored_item) | |
| meta_results.sort(key=lambda x: x.get('_score', 0), reverse=True) | |
| top_meta = meta_results[:k] | |
| final_results = [] | |
| seen = set() | |
| def _build_result(item_data: Dict, match_type: str, score: int = 0) -> Dict: | |
| result = { | |
| 'source': source_label, | |
| 'type': match_type, | |
| 'id': item_data.get('id'), | |
| 'title': item_data.get(title_field), | |
| 'content': item_data.get(content_field), | |
| 'link': item_data.get(link_field), | |
| } | |
| if date_field: | |
| result['date'] = item_data.get(date_field, '') | |
| if score: | |
| result['_score'] = score | |
| return result | |
| # High Confidence | |
| for item in top_meta: | |
| if item.get('_score', 0) >= SCORING.high_confidence_threshold: | |
| rid = item.get('id') | |
| if rid in seen: continue | |
| final_results.append(_build_result(item, 'Metadata Match (High Confidence)', item.get('_score', 0))) | |
| seen.add(rid) | |
| # Medium Confidence (remainder) | |
| for item in top_meta: | |
| rid = item.get('id') | |
| if rid in seen: continue | |
| final_results.append(_build_result(item, 'Metadata Match (Medium Confidence)', item.get('_score', 0))) | |
| seen.add(rid) | |
| return final_results | |
| def _calculate_fuzzy_score(self, text: str, keywords: List[str]) -> int: | |
| if not keywords: return 0 | |
| scores = [] | |
| for k in keywords: | |
| if len(k) < 2: continue | |
| score = fuzz.partial_ratio(k.lower(), text.lower()) | |
| scores.append(score) | |
| if not scores: return 0 | |
| # Base score is the best individual match | |
| max_score = max(scores) | |
| # [Multi-Keyword Bonus] | |
| # If multiple unique keywords match significantly, boost the total score. | |
| # This helps "Shane UT" rank Shane's UT documents higher than other Shane docs. | |
| significant_matches = [s for s in scores if s >= 70] | |
| if len(significant_matches) > 1: | |
| max_score += (len(significant_matches) - 1) * SCORING.multi_keyword_bonus | |
| return min(max_score, SCORING.max_score_cap) | |
| def search_internal(self, keywords: List[str], k: int = 100, filters: Dict = None, min_score: int = 80) -> List[Dict]: | |
| """Search Internal Research (Fuzzy)""" | |
| # 1. Fuzzy Search | |
| meta_results = [] | |
| # [Synonym Expansion] Use shared method | |
| search_keywords = self._expand_keywords(keywords) | |
| # [Stop Words] | |
| # Generic terms that should not trigger the Title Specific Boost | |
| STOP_WORDS = ["리μμΉ", "λ³΄κ³ μ", "μ‘°μ¬", "κ²°κ³Όλ¬Ό", "κ²°κ³Ό", "νμΌ"] | |
| for item in self.internal_data: | |
| meta = item.get('metadata', {}) | |
| detail = item.get('μμΈ μ 보', {}) | |
| res_id = item.get('id', '') | |
| # Exact ID - Force Boost to Top | |
| if any(kw.upper() == res_id.upper() for kw in keywords): | |
| scored_item = dict(item) | |
| scored_item['_score'] = SCORING.exact_id_boost # Massive boost for explicit ID | |
| meta_results.insert(0, scored_item) | |
| continue | |
| text = f"{meta.get('리μμΉ μ λͺ©', '')} {meta.get('νκ·Έ', '')} {meta.get('λ΄λΉ 리μμ²', '')} {meta.get('λ°©λ²λ‘ ', '')} {meta.get('λλ©μΈ', '')} {detail.get('μ£Όμ λ΄μ© μμ½', '')}" | |
| score = self._calculate_fuzzy_score(text, search_keywords) | |
| # [Automatic Boosting based on Keywords] | |
| # Even if 'filters' is not provided, check if keywords match Researcher or Methodology fields | |
| item_researcher = meta.get('λ΄λΉ 리μμ²', '') | |
| item_method = meta.get('λ°©λ²λ‘ ', '') | |
| item_date = meta.get('λ μ§', '') | |
| # 1. Researcher Boost | |
| # Check if any keyword matches the researcher name significantly | |
| for kw in search_keywords: | |
| if len(kw) < 2: continue | |
| if fuzz.partial_ratio(kw.lower(), item_researcher.lower()) >= SCORING.fuzzy_researcher_threshold: | |
| score += SCORING.researcher_boost # Boost for researcher match | |
| break | |
| # 2. Methodology Boost | |
| for kw in search_keywords: | |
| if len(kw) < 2: continue | |
| if fuzz.partial_ratio(kw.lower(), item_method.lower()) >= SCORING.fuzzy_method_threshold: | |
| score += SCORING.methodology_boost # Boost for methodology match | |
| break | |
| # 3. Title Specific Boost (NEW) | |
| item_title = meta.get('리μμΉ μ λͺ©', '') | |
| for kw in search_keywords: | |
| if len(kw) < 2: continue | |
| # [Stop Word Check] Skip boost for generic terms like "리μμΉ" | |
| if kw in STOP_WORDS: continue | |
| # [Title Match Bonus] | |
| # If a keyword matches the TITLE significantly, it's likely very relevant. | |
| if fuzz.partial_ratio(kw.lower(), item_title.lower()) >= SCORING.fuzzy_title_threshold: | |
| score += SCORING.title_boost # Strong boost for title match | |
| break | |
| # Methodology Boosting (Legacy Filter Support) | |
| if filters: | |
| if filters.get('methodology'): | |
| target_method = filters['methodology'] | |
| if fuzz.partial_ratio(target_method.lower(), item_method.lower()) >= 80: | |
| score += SCORING.filter_force_boost # Force boost to top | |
| # Researcher Boosting | |
| if filters.get('researcher'): | |
| target_researcher = filters['researcher'] | |
| # Fuzzy match for names (e.g. "Yondu" vs "Yondu Kim") | |
| if fuzz.partial_ratio(target_researcher.lower(), item_researcher.lower()) >= SCORING.min_score: | |
| score += SCORING.researcher_boost # Significant boost | |
| # Date Boosting | |
| if filters.get('date'): | |
| target_date = filters['date'] # Expected YYYY | |
| if target_date in item_date: | |
| score += 200 | |
| if score >= min_score: # Use dynamic threshold (Default 80) | |
| scored_item = dict(item) | |
| scored_item['_score'] = score | |
| meta_results.append(scored_item) | |
| meta_results.sort(key=lambda x: x.get('_score', 0), reverse=True) | |
| if k is None: k = 100 | |
| k = int(k) | |
| top_meta = meta_results[:k] | |
| final_results = [] | |
| seen = set() | |
| def _build_internal_result(item, match_type): | |
| meta = item.get('metadata', {}) | |
| detail = item.get('μμΈ μ 보', {}) | |
| related = detail.get('κ΄λ ¨ μλ£', []) | |
| links_str = "" | |
| if related: | |
| links_str = "\nRelated Materials:\n" + "\n".join([f"- {l.get('μ λͺ©')}: {l.get('url')}" for l in related]) | |
| return { | |
| 'source': 'INTERNAL', | |
| 'type': match_type, | |
| 'id': item.get('id'), | |
| 'title': meta.get('리μμΉ μ λͺ©'), | |
| 'date': meta.get('λ μ§', 'N/A'), | |
| 'researcher': meta.get('λ΄λΉ 리μμ²', 'N/A'), | |
| 'methodology': meta.get('λ°©λ²λ‘ ', 'N/A'), | |
| 'domain': meta.get('λλ©μΈ', 'N/A'), | |
| 'content': detail.get('μ£Όμ λ΄μ© μμ½'), | |
| 'link': f"{meta.get('κ²°κ³Ό λ§ν¬')}{links_str}", | |
| '_score': item.get('_score', 0) | |
| } | |
| # High Confidence | |
| for item in top_meta: | |
| if item.get('_score', 0) >= SCORING.high_confidence_threshold: | |
| rid = item.get('id') | |
| if rid in seen: continue | |
| final_results.append(_build_internal_result(item, 'Metadata Match (High Confidence)')) | |
| seen.add(rid) | |
| # Medium Confidence (remainder) | |
| for item in top_meta: | |
| rid = item.get('id') | |
| if rid in seen: continue | |
| final_results.append(_build_internal_result(item, 'Metadata Match (Medium Confidence)')) | |
| seen.add(rid) | |
| # 3. Global Sort by Score (Primary) and Date (Secondary) | |
| def sort_key(item): | |
| # Primary: Score (DESC) | |
| score = item.get('_score', 0) | |
| # Secondary: Date (DESC) | |
| d_str = item.get('date', '') | |
| clean_d = re.sub(r'[^0-9]', '', str(d_str)) | |
| date_val = 0 | |
| try: | |
| if clean_d: date_val = int(clean_d) | |
| except ValueError: | |
| pass | |
| return (score, date_val) | |
| final_results.sort(key=sort_key, reverse=True) | |
| return final_results | |
| def search_baymard(self, keywords: List[str], k=5, min_score: int = 80) -> List[Dict]: | |
| """Search Baymard Guidelines""" | |
| return self._search_external( | |
| source_label='BAYMARD', data=self.baymard_data, | |
| keywords=keywords, | |
| text_fields=['title', 'description', 'issue'], | |
| title_field='title', content_field='advice', link_field='url', | |
| k=k, min_score=min_score, | |
| ) | |
| def search_nng(self, keywords: List[str], k=5, min_score: int = 80) -> List[Dict]: | |
| """Search NN/g Articles""" | |
| return self._search_external( | |
| source_label='NNG', data=self.nng_data, | |
| keywords=keywords, | |
| text_fields=['title', 'description'], | |
| title_field='title', content_field='description', link_field='url', | |
| k=k, min_score=min_score, | |
| date_field='date', | |
| ) | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Hybrid Search (Fuzzy + Semantic via RRF) | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| def hybrid_search( | |
| self, | |
| keywords: List[str], | |
| query: str, | |
| source: Optional[str] = None, | |
| k: int = 10, | |
| min_score: int = 60, | |
| ) -> List[Dict]: | |
| """Combine fuzzy and semantic search results using Reciprocal Rank Fusion. | |
| Args: | |
| keywords: Keywords for fuzzy search. | |
| query: Natural language query for semantic search. | |
| source: Optional source filter ("INTERNAL", "BAYMARD", "NNG"). | |
| k: Max results to return. | |
| min_score: Minimum fuzzy score threshold. | |
| Returns: | |
| Merged results sorted by RRF score, each dict includes 'match_type'. | |
| """ | |
| rrf_k = SCORING.rrf_k | |
| sources_to_search = [source.upper()] if source else ["INTERNAL", "BAYMARD", "NNG"] | |
| # 1. Collect fuzzy results | |
| fuzzy_results = [] | |
| for src in sources_to_search: | |
| if src == "INTERNAL": | |
| fuzzy_results.extend(self.search_internal(keywords, k=k, min_score=min_score)) | |
| elif src == "BAYMARD": | |
| fuzzy_results.extend(self.search_baymard(keywords, k=k, min_score=min_score)) | |
| elif src == "NNG": | |
| fuzzy_results.extend(self.search_nng(keywords, k=k, min_score=min_score)) | |
| # 2. Collect semantic results | |
| semantic_hits = [] | |
| if self.vector_store and query: | |
| semantic_hits = self.vector_store.semantic_search( | |
| query=query, k=k, source_filter=source | |
| ) | |
| # If no semantic results, return fuzzy results as-is | |
| if not semantic_hits: | |
| for r in fuzzy_results: | |
| r["match_type"] = "Fuzzy Match" | |
| return fuzzy_results[:k] | |
| # 3. Build rank maps (doc_id β rank, 0-indexed) | |
| fuzzy_rank = {} | |
| for rank, item in enumerate(fuzzy_results): | |
| doc_id = str(item.get("id", "")) | |
| src = item.get("source", "") | |
| key = f"{src}:{doc_id}" | |
| if key not in fuzzy_rank: | |
| fuzzy_rank[key] = rank | |
| semantic_rank = {} | |
| for rank, hit in enumerate(semantic_hits): | |
| key = f"{hit['source']}:{hit['doc_id']}" | |
| if key not in semantic_rank: | |
| semantic_rank[key] = rank | |
| # 4. Compute RRF scores | |
| all_keys = set(fuzzy_rank.keys()) | set(semantic_rank.keys()) | |
| rrf_scores = {} | |
| for key in all_keys: | |
| score = 0.0 | |
| if key in fuzzy_rank: | |
| score += 1.0 / (rrf_k + fuzzy_rank[key]) | |
| if key in semantic_rank: | |
| score += 1.0 / (rrf_k + semantic_rank[key]) | |
| rrf_scores[key] = score | |
| # 5. Build result list | |
| # Map fuzzy results by key for quick lookup | |
| fuzzy_map = {} | |
| for item in fuzzy_results: | |
| key = f"{item.get('source', '')}:{item.get('id', '')}" | |
| if key not in fuzzy_map: | |
| fuzzy_map[key] = item | |
| sorted_keys = sorted(rrf_scores.keys(), key=lambda k: rrf_scores[k], reverse=True) | |
| merged = [] | |
| for key in sorted_keys[:k]: | |
| in_fuzzy = key in fuzzy_rank | |
| in_semantic = key in semantic_rank | |
| if in_fuzzy and in_semantic: | |
| match_type = "Hybrid Match" | |
| elif in_fuzzy: | |
| match_type = "Fuzzy Match" | |
| else: | |
| match_type = "Semantic Match" | |
| if key in fuzzy_map: | |
| result = dict(fuzzy_map[key]) | |
| result["match_type"] = match_type | |
| merged.append(result) | |
| else: | |
| # Semantic-only hit: build result from document data | |
| result = self._build_result_from_semantic(key) | |
| if result: | |
| result["match_type"] = match_type | |
| merged.append(result) | |
| return merged | |
| def _build_result_from_semantic(self, key: str) -> Optional[Dict]: | |
| """Build a search result dict for a semantic-only hit.""" | |
| parts = key.split(":", 1) | |
| if len(parts) != 2: | |
| return None | |
| source, doc_id = parts | |
| doc = self.get_document_by_id(doc_id, source) | |
| if not doc: | |
| return None | |
| if source == "INTERNAL": | |
| return { | |
| "source": "INTERNAL", | |
| "type": "Semantic Match", | |
| "id": doc_id, | |
| "title": doc.get("title", ""), | |
| "date": doc.get("date", ""), | |
| "researcher": doc.get("researcher", ""), | |
| "methodology": doc.get("methodology", ""), | |
| "domain": doc.get("domain", ""), | |
| "content": doc.get("summary", ""), | |
| "link": doc.get("link", ""), | |
| } | |
| elif source == "BAYMARD": | |
| return { | |
| "source": "BAYMARD", | |
| "type": "Semantic Match", | |
| "id": doc_id, | |
| "title": doc.get("title", ""), | |
| "content": doc.get("advice", "") or doc.get("full_content", ""), | |
| "link": doc.get("link", ""), | |
| } | |
| elif source == "NNG": | |
| return { | |
| "source": "NNG", | |
| "type": "Semantic Match", | |
| "id": doc_id, | |
| "title": doc.get("title", ""), | |
| "content": doc.get("description", ""), | |
| "link": doc.get("link", ""), | |
| "date": doc.get("date", ""), | |
| } | |
| return None | |
| def get_specific_item(self, query: str) -> Dict[str, Any]: | |
| """ | |
| Finds a specific item (Internal/Baymard/NNG) best matching the query (Title/ID/Link). | |
| Returns Dict with 'content' (FULL), 'title', 'source', etc. or None if not found. | |
| """ | |
| # 1. Normalize query | |
| query_norm = query.strip().lower() | |
| best_match = None | |
| best_score = 0 | |
| # Helper to check match | |
| def check_match(item, source_type): | |
| nonlocal best_match, best_score | |
| # Fields to check | |
| rid = str(item.get('id', '')) | |
| # Handle different data structures | |
| if source_type == 'INTERNAL': | |
| meta = item.get('metadata', {}) | |
| title = meta.get('리μμΉ μ λͺ©', '') | |
| link = meta.get('κ²°κ³Ό λ§ν¬', '') | |
| content = item.get('μμΈ μ 보', {}).get('μ£Όμ λ΄μ© μμ½', '') # Will enrich later | |
| elif source_type == 'BAYMARD': | |
| title = item.get('title', '') | |
| link = item.get('url', '') | |
| content = item.get('advice', '') | |
| elif source_type == 'NNG': | |
| title = item.get('title', '') | |
| link = item.get('url', '') | |
| content = item.get('description', '') | |
| # Calculate Score | |
| # Exact ID Match | |
| if rid.lower() in query_norm: | |
| score = 100 | |
| # URL Match | |
| elif query_norm in link.lower() and len(query_norm) > 10: | |
| score = 100 | |
| # Title Fuzzy Match | |
| else: | |
| score = fuzz.ratio(query_norm, title.lower()) | |
| # Partial match for long titles if query is significant part | |
| if len(query_norm) > 5: | |
| partial = fuzz.partial_ratio(query_norm, title.lower()) | |
| if partial > score: score = partial | |
| if score > 85 and score > best_score: | |
| best_score = score | |
| best_match = { | |
| "source": source_type, | |
| "id": rid, | |
| "title": title, | |
| "link": link, | |
| "content": content, # Placeholder, will fetch full doc if needed | |
| "original_item": item | |
| } | |
| # Scan Data sources | |
| for item in self.internal_data: check_match(item, 'INTERNAL') | |
| for item in self.baymard_data: check_match(item, 'BAYMARD') | |
| for item in self.nng_data: check_match(item, 'NNG') | |
| if best_match: | |
| logger.info(f"Specific Item Match: {best_match['title']} ({best_match['source']}) - Score: {best_score}") | |
| return best_match | |
| return None | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Agentic Retriever Helper Methods | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _read_md_file(self, file_path: str) -> str: | |
| """Read a markdown file from DATA_DIR. Returns content or empty string.""" | |
| try: | |
| full_path = os.path.join(str(DATA_DIR), file_path.lstrip('/')) | |
| if os.path.exists(full_path): | |
| with open(full_path, 'r', encoding='utf-8') as f: | |
| return f.read() | |
| except Exception as e: | |
| logger.warning(f"Failed to read MD file {file_path}: {e}") | |
| return "" | |
| def get_document_by_id(self, doc_id: str, source: str) -> Optional[Dict[str, Any]]: | |
| """Returns full document content by ID and source. Used by agentic retriever tools.""" | |
| source = source.upper() | |
| if source == "INTERNAL": | |
| item = next((d for d in self.internal_data if d.get('id') == doc_id), None) | |
| if not item: | |
| return None | |
| meta = item.get('metadata', {}) | |
| detail = item.get('μμΈ μ 보', {}) | |
| # Read full MD content if available | |
| file_path = meta.get('file_path', '') | |
| md_content = self._read_md_file(file_path) if file_path and file_path != "νμΌ μμ" else "" | |
| related = detail.get('κ΄λ ¨ μλ£', []) | |
| links_str = "" | |
| if related: | |
| links_str = "\nκ΄λ ¨ μλ£:\n" + "\n".join([f"- {l.get('μ λͺ©')}: {l.get('url')}" for l in related]) | |
| return { | |
| 'id': doc_id, | |
| 'source': 'INTERNAL', | |
| 'title': meta.get('리μμΉ μ λͺ©', ''), | |
| 'date': meta.get('λ μ§', ''), | |
| 'researcher': meta.get('λ΄λΉ 리μμ²', ''), | |
| 'methodology': meta.get('λ°©λ²λ‘ ', ''), | |
| 'domain': meta.get('λλ©μΈ', ''), | |
| 'tags': meta.get('νκ·Έ', ''), | |
| 'summary': detail.get('μ£Όμ λ΄μ© μμ½', ''), | |
| 'full_content': md_content or detail.get('μ£Όμ λ΄μ© μμ½', ''), | |
| 'link': meta.get('κ²°κ³Ό λ§ν¬', '') + links_str, | |
| } | |
| elif source == "BAYMARD": | |
| item = next((d for d in self.baymard_data if str(d.get('id')) == str(doc_id)), None) | |
| if not item: | |
| return None | |
| return { | |
| 'id': str(item.get('id')), | |
| 'source': 'BAYMARD', | |
| 'title': item.get('title', ''), | |
| 'topic': item.get('topic', ''), | |
| 'issue': item.get('issue', ''), | |
| 'advice': item.get('advice', ''), | |
| 'description': item.get('description', ''), | |
| 'full_content': f"Issue:\n{item.get('issue', '')}\n\nAdvice:\n{item.get('advice', '')}\n\nDescription:\n{item.get('description', '')}", | |
| 'link': item.get('url', ''), | |
| } | |
| elif source == "NNG": | |
| item = next((d for d in self.nng_data if d.get('id') == doc_id), None) | |
| if not item: | |
| return None | |
| contents_list = item.get('contents', []) | |
| contents_str = "\n".join(f"- {c}" for c in contents_list) if isinstance(contents_list, list) else str(contents_list) | |
| return { | |
| 'id': doc_id, | |
| 'source': 'NNG', | |
| 'title': item.get('title', ''), | |
| 'date': item.get('date', ''), | |
| 'authors': item.get('authors', []), | |
| 'topics': item.get('topics', []), | |
| 'description': item.get('description', ''), | |
| 'contents_outline': contents_str, | |
| 'full_content': f"Description:\n{item.get('description', '')}\n\nContents:\n{contents_str}", | |
| 'link': item.get('url', ''), | |
| } | |
| return None | |
| def list_topics(self, source: str) -> List[Dict[str, Any]]: | |
| """Returns list of topics with document counts for a given source.""" | |
| source = source.upper() | |
| if source == "BAYMARD": | |
| counter = Counter(item.get('topic', 'Unknown') for item in self.baymard_data) | |
| return [{'topic': topic, 'count': count} for topic, count in counter.most_common()] | |
| elif source == "NNG": | |
| counter: Counter = Counter() | |
| for item in self.nng_data: | |
| for t in item.get('topics', []): | |
| counter[t] += 1 | |
| return [{'topic': topic, 'count': count} for topic, count in counter.most_common()] | |
| elif source == "INTERNAL": | |
| counter = Counter(item.get('metadata', {}).get('λλ©μΈ', 'Unknown') for item in self.internal_data) | |
| return [{'topic': topic, 'count': count} for topic, count in counter.most_common()] | |
| return [] | |
| def list_docs_by_topic(self, source: str, topic: str) -> List[Dict[str, Any]]: | |
| """Returns document ID/title/preview for a given source and topic.""" | |
| source = source.upper() | |
| results = [] | |
| if source == "BAYMARD": | |
| for item in self.baymard_data: | |
| if item.get('topic', '').lower() == topic.lower(): | |
| results.append({ | |
| 'id': str(item.get('id')), | |
| 'title': item.get('title', ''), | |
| 'preview': (item.get('description', '') or item.get('issue', ''))[:120], | |
| }) | |
| elif source == "NNG": | |
| for item in self.nng_data: | |
| if topic.lower() in [t.lower() for t in item.get('topics', [])]: | |
| results.append({ | |
| 'id': item.get('id', ''), | |
| 'title': item.get('title', ''), | |
| 'preview': (item.get('description', ''))[:120], | |
| }) | |
| elif source == "INTERNAL": | |
| for item in self.internal_data: | |
| if item.get('metadata', {}).get('λλ©μΈ', '').lower() == topic.lower(): | |
| meta = item.get('metadata', {}) | |
| results.append({ | |
| 'id': item.get('id', ''), | |
| 'title': meta.get('리μμΉ μ λͺ©', ''), | |
| 'preview': (item.get('μμΈ μ 보', {}).get('μ£Όμ λ΄μ© μμ½', ''))[:120], | |
| }) | |
| return results | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Aggregation | |
| # ββββββββββββββββββββββββββββββββββββββββββββββ | |
| _INTERNAL_GROUP_FIELDS = { | |
| "year": lambda item: item.get('metadata', {}).get('λ μ§', '')[:4] or 'Unknown', | |
| "domain": lambda item: item.get('metadata', {}).get('λλ©μΈ', '') or 'Unknown', | |
| "methodology": lambda item: item.get('metadata', {}).get('λ°©λ²λ‘ ', '') or 'Unknown', | |
| "researcher": lambda item: item.get('metadata', {}).get('λ΄λΉ 리μμ²', '') or 'Unknown', | |
| "tag": lambda item: item.get('metadata', {}).get('νκ·Έ', '') or 'Unknown', | |
| } | |
| _BAYMARD_GROUP_FIELDS = { | |
| "topic": lambda item: [item.get('topic', 'Unknown')], | |
| "theme": lambda item: [item.get('theme', 'Unknown')], | |
| "platform": lambda item: item.get('platforms', []) or ['Unknown'], | |
| "importance": lambda item: [str(item.get('importance', 'Unknown'))], | |
| } | |
| _NNG_GROUP_FIELDS = { | |
| "topic": lambda item: item.get('topics', []) or ['Unknown'], | |
| "author": lambda item: item.get('authors', []) or ['Unknown'], | |
| "year": lambda item: [Retriever._extract_year_from_text(item.get('date', ''))], | |
| } | |
| def _extract_year_from_text(date_text: str) -> str: | |
| """Extract a 4-digit year from free-form date text like 'June 25, 2023'.""" | |
| if not date_text: | |
| return 'Unknown' | |
| m = re.search(r'\b(19|20)\d{2}\b', date_text) | |
| return m.group(0) if m else 'Unknown' | |
| def aggregate(self, source: str, group_by: str, | |
| filter_field: str = None, filter_value: str = None) -> Dict: | |
| """Aggregate document counts by a metadata field, with optional filtering. | |
| Args: | |
| source: "INTERNAL", "BAYMARD", or "NNG". | |
| group_by: Field name to group by (source-specific). | |
| filter_field: Optional field to filter on before aggregating. | |
| filter_value: Value for filter (case-insensitive partial match). | |
| Returns: | |
| Dict with source, group_by, total_docs, counts (list of (value, count) tuples), | |
| and optional filter info. | |
| """ | |
| source = source.upper() | |
| # ALL: aggregate across all sources, grouped by source name | |
| if source == "ALL": | |
| counts = [ | |
| ("INTERNAL", len(self.internal_data)), | |
| ("BAYMARD", len(self.baymard_data)), | |
| ("NNG", len(self.nng_data)), | |
| ] | |
| total = sum(c for _, c in counts) | |
| return { | |
| "source": "ALL", | |
| "group_by": "source", | |
| "total_docs": total, | |
| "counts": counts, | |
| "filter": None, | |
| } | |
| # Select data and field maps | |
| if source == "INTERNAL": | |
| data = self.internal_data | |
| group_map = self._INTERNAL_GROUP_FIELDS | |
| filter_map = self._INTERNAL_GROUP_FIELDS | |
| elif source == "BAYMARD": | |
| data = self.baymard_data | |
| group_map = self._BAYMARD_GROUP_FIELDS | |
| filter_map = self._BAYMARD_GROUP_FIELDS | |
| elif source == "NNG": | |
| data = self.nng_data | |
| group_map = self._NNG_GROUP_FIELDS | |
| filter_map = self._NNG_GROUP_FIELDS | |
| else: | |
| return {"error": f"Unknown source: {source}. Use INTERNAL, BAYMARD, NNG, or ALL."} | |
| if group_by not in group_map: | |
| valid = ", ".join(group_map.keys()) | |
| return {"error": f"Invalid group_by '{group_by}' for {source}. Valid: {valid}"} | |
| # Validate filter_field | |
| if filter_field and filter_field not in filter_map: | |
| valid = ", ".join(filter_map.keys()) | |
| return {"error": f"Invalid filter_field '{filter_field}' for {source}. Valid: {valid}"} | |
| group_fn = group_map[group_by] | |
| # Apply optional filter | |
| filtered_data = data | |
| if filter_field and filter_value: | |
| filter_fn = filter_map[filter_field] | |
| fv_lower = filter_value.lower() | |
| filtered_data = [] | |
| for item in data: | |
| values = filter_fn(item) | |
| if isinstance(values, str): | |
| values = [values] | |
| if any(fv_lower in v.lower() for v in values): | |
| filtered_data.append(item) | |
| # Count | |
| counter: Counter = Counter() | |
| for item in filtered_data: | |
| values = group_fn(item) | |
| if isinstance(values, str): | |
| values = [values] | |
| for v in values: | |
| counter[v] += 1 | |
| return { | |
| "source": source, | |
| "group_by": group_by, | |
| "total_docs": len(filtered_data), | |
| "counts": counter.most_common(), | |
| "filter": {"field": filter_field, "value": filter_value} if filter_field else None, | |
| } | |
| def filter_by_metadata(self, source: str, field: str, value: str, | |
| max_results: int = 20) -> List[Dict]: | |
| """Filter documents by metadata field and return matching documents with previews. | |
| Args: | |
| source: "INTERNAL", "BAYMARD", or "NNG". | |
| field: Metadata field to filter on (same as aggregate group_by fields). | |
| value: Filter value (case-insensitive partial match). | |
| max_results: Maximum documents to return. | |
| Returns: | |
| List of matching documents with id, title, preview, and metadata. | |
| """ | |
| source = source.upper() | |
| field_maps = { | |
| "INTERNAL": (self.internal_data, self._INTERNAL_GROUP_FIELDS), | |
| "BAYMARD": (self.baymard_data, self._BAYMARD_GROUP_FIELDS), | |
| "NNG": (self.nng_data, self._NNG_GROUP_FIELDS), | |
| } | |
| if source not in field_maps: | |
| return [] | |
| data, group_map = field_maps[source] | |
| if field not in group_map: | |
| return [] | |
| filter_fn = group_map[field] | |
| val_lower = value.lower() | |
| results = [] | |
| for item in data: | |
| values = filter_fn(item) | |
| if isinstance(values, str): | |
| values = [values] | |
| if not any(val_lower in v.lower() for v in values): | |
| continue | |
| if source == "INTERNAL": | |
| meta = item.get('metadata', {}) | |
| detail = item.get('μμΈ μ 보', {}) | |
| results.append({ | |
| 'id': item.get('id', ''), | |
| 'title': meta.get('리μμΉ μ λͺ©', ''), | |
| 'source': 'INTERNAL', | |
| 'date': meta.get('λ μ§', ''), | |
| 'domain': meta.get('λλ©μΈ', ''), | |
| 'methodology': meta.get('λ°©λ²λ‘ ', ''), | |
| 'researcher': meta.get('λ΄λΉ 리μμ²', ''), | |
| 'tags': meta.get('νκ·Έ', ''), | |
| 'link': meta.get('κ²°κ³Ό λ§ν¬', ''), | |
| 'preview': (detail.get('μ£Όμ λ΄μ© μμ½', ''))[:250], | |
| }) | |
| elif source == "BAYMARD": | |
| results.append({ | |
| 'id': item.get('id', ''), | |
| 'title': item.get('title', ''), | |
| 'source': 'BAYMARD', | |
| 'topic': item.get('topic', ''), | |
| 'importance': item.get('importance', ''), | |
| 'link': item.get('url', ''), | |
| 'preview': (item.get('description', ''))[:250], | |
| }) | |
| elif source == "NNG": | |
| results.append({ | |
| 'id': item.get('id', ''), | |
| 'title': item.get('title', ''), | |
| 'source': 'NNG', | |
| 'authors': item.get('authors', []), | |
| 'topics': item.get('topics', []), | |
| 'link': item.get('url', ''), | |
| 'preview': (item.get('description', ''))[:250], | |
| }) | |
| if len(results) >= max_results: | |
| break | |
| return results | |