debug-log
Claude Opus 4.6
feat: Insight Repository β€” GitLab μ΄ˆκΈ°ν™” + HF μžλ™ 배포
501bc8c
Raw History Blame Contribute Delete
38.1 kB
import os
import logging
import re
from collections import Counter
from typing import List, Dict, Any, Optional
from thefuzz import fuzz
from .config import config, SCORING, DATA_DIR
from .data_loader import load_internal_research, load_baymard_guidelines, load_nng_articles
from .vector_store import VectorStore
logger = logging.getLogger("Retriever")
# ──────────────────────────────────────────────
# Internal-Only Synonym Dictionary
# Contains ONLY company-specific abbreviations and names that an LLM cannot infer.
# General domain terms (CTR, μ΄νƒˆ, 결제, etc.) are handled by LLM-based query expansion.
# ──────────────────────────────────────────────
SYNONYMS = {
# ── λ¦¬μ„œμ²˜ 이름 (사내 인물) ──
"shane": ["shane ahn", "쉐인"],
"sonya": ["μ†Œλƒ"],
"judy": ["μ£Όλ””"],
"μš˜λ‘": ["yondu"],
"라일리": ["riley"],
"쟌": ["jan"],
# ── 사내 μ„œλΉ„μŠ€/도ꡬ μ•½μ–΄ ──
"νŒŒνŠΈλ„ˆμ„Όν„°": ["νŒŒνŠΈλ„ˆ μ„Όν„°", "ν˜‘λ ₯사 μ„Όν„°", "제휴점 μ„Όν„°", "사μž₯λ‹˜ μ‚¬μ΄νŠΈ"],
"bh": ["booking hub", "λΆ€ν‚Ήν—ˆλΈŒ"],
"rh": ["resort hub", "λ¦¬μ‘°νŠΈν—ˆλΈŒ"],
"gc": ["global commerce", "κΈ€λ‘œλ²Œμ»€λ¨ΈμŠ€"],
}
# Precomputed reverse lookup: synonym_value β†’ canonical_key
# Built once at module load instead of iterating SYNONYMS.items() on every call.
_SYNONYM_REVERSE: Dict[str, str] = {}
for _key, _syns in SYNONYMS.items():
for _syn in _syns:
_SYNONYM_REVERSE[_syn.lower()] = _key
class Retriever:
def __init__(self):
logger.info("Initializing Retriever...")
# Load Data
self.internal_data = load_internal_research()
self.baymard_data = load_baymard_guidelines()
self.nng_data = load_nng_articles()
logger.info(f"Data Loaded: {len(self.internal_data)} Internal, {len(self.baymard_data)} Baymard, {len(self.nng_data)} NN/g")
# Initialize VectorStore for semantic search (graceful degradation)
self.vector_store: Optional[VectorStore] = None
if SCORING.semantic_search_enabled:
try:
self.vector_store = VectorStore(
self.internal_data, self.baymard_data, self.nng_data
)
logger.info("VectorStore initialized for semantic search.")
except Exception as e:
logger.warning("VectorStore init failed, falling back to fuzzy-only: %s", e)
self.vector_store = None
logger.info("Retriever Ready.")
def _expand_keywords(self, keywords: List[str]) -> List[str]:
"""
Expands keywords using the shared SYNONYMS dictionary.
Supports: direct match, reverse match (via precomputed map), and space normalization.
Shared across search_internal, search_baymard, search_nng.
"""
expanded = set(keywords)
for kw in keywords:
k_lower = kw.lower()
# 1. Direct match: keyword is a key in SYNONYMS
if k_lower in SYNONYMS:
for syn in SYNONYMS[k_lower]:
expanded.add(syn)
# 2. Reverse match: O(1) lookup via precomputed map
if k_lower in _SYNONYM_REVERSE:
expanded.add(_SYNONYM_REVERSE[k_lower])
# 3. Space normalization (Korean spacing)
if " " in kw:
expanded.add(kw.replace(" ", ""))
return list(expanded)
def _search_external(self, source_label: str, data: List[Dict],
keywords: List[str], text_fields: List[str],
title_field: str, content_field: str, link_field: str,
k: int = 5, min_score: int = 80,
date_field: str = None) -> List[Dict]:
"""
Fuzzy search for external sources (Baymard, NN/g).
"""
search_keywords = self._expand_keywords(keywords)
meta_results = []
for item in data:
text = " ".join(item.get(f, '') for f in text_fields)
score = self._calculate_fuzzy_score(text, search_keywords)
# Title Boost
item_title = item.get(title_field, '')
for kw in search_keywords:
if len(kw) < 2: continue
if fuzz.partial_ratio(kw.lower(), item_title.lower()) >= SCORING.fuzzy_title_threshold:
score += SCORING.title_boost
break
if score >= min_score:
scored_item = dict(item)
scored_item['_score'] = score
meta_results.append(scored_item)
meta_results.sort(key=lambda x: x.get('_score', 0), reverse=True)
top_meta = meta_results[:k]
final_results = []
seen = set()
def _build_result(item_data: Dict, match_type: str, score: int = 0) -> Dict:
result = {
'source': source_label,
'type': match_type,
'id': item_data.get('id'),
'title': item_data.get(title_field),
'content': item_data.get(content_field),
'link': item_data.get(link_field),
}
if date_field:
result['date'] = item_data.get(date_field, '')
if score:
result['_score'] = score
return result
# High Confidence
for item in top_meta:
if item.get('_score', 0) >= SCORING.high_confidence_threshold:
rid = item.get('id')
if rid in seen: continue
final_results.append(_build_result(item, 'Metadata Match (High Confidence)', item.get('_score', 0)))
seen.add(rid)
# Medium Confidence (remainder)
for item in top_meta:
rid = item.get('id')
if rid in seen: continue
final_results.append(_build_result(item, 'Metadata Match (Medium Confidence)', item.get('_score', 0)))
seen.add(rid)
return final_results
def _calculate_fuzzy_score(self, text: str, keywords: List[str]) -> int:
if not keywords: return 0
scores = []
for k in keywords:
if len(k) < 2: continue
score = fuzz.partial_ratio(k.lower(), text.lower())
scores.append(score)
if not scores: return 0
# Base score is the best individual match
max_score = max(scores)
# [Multi-Keyword Bonus]
# If multiple unique keywords match significantly, boost the total score.
# This helps "Shane UT" rank Shane's UT documents higher than other Shane docs.
significant_matches = [s for s in scores if s >= 70]
if len(significant_matches) > 1:
max_score += (len(significant_matches) - 1) * SCORING.multi_keyword_bonus
return min(max_score, SCORING.max_score_cap)
def search_internal(self, keywords: List[str], k: int = 100, filters: Dict = None, min_score: int = 80) -> List[Dict]:
"""Search Internal Research (Fuzzy)"""
# 1. Fuzzy Search
meta_results = []
# [Synonym Expansion] Use shared method
search_keywords = self._expand_keywords(keywords)
# [Stop Words]
# Generic terms that should not trigger the Title Specific Boost
STOP_WORDS = ["λ¦¬μ„œμΉ˜", "λ³΄κ³ μ„œ", "쑰사", "κ²°κ³Όλ¬Ό", "κ²°κ³Ό", "파일"]
for item in self.internal_data:
meta = item.get('metadata', {})
detail = item.get('상세 정보', {})
res_id = item.get('id', '')
# Exact ID - Force Boost to Top
if any(kw.upper() == res_id.upper() for kw in keywords):
scored_item = dict(item)
scored_item['_score'] = SCORING.exact_id_boost # Massive boost for explicit ID
meta_results.insert(0, scored_item)
continue
text = f"{meta.get('λ¦¬μ„œμΉ˜ 제λͺ©', '')} {meta.get('νƒœκ·Έ', '')} {meta.get('λ‹΄λ‹Ή λ¦¬μ„œμ²˜', '')} {meta.get('방법둠', '')} {meta.get('도메인', '')} {detail.get('μ£Όμš” λ‚΄μš© μš”μ•½', '')}"
score = self._calculate_fuzzy_score(text, search_keywords)
# [Automatic Boosting based on Keywords]
# Even if 'filters' is not provided, check if keywords match Researcher or Methodology fields
item_researcher = meta.get('λ‹΄λ‹Ή λ¦¬μ„œμ²˜', '')
item_method = meta.get('방법둠', '')
item_date = meta.get('λ‚ μ§œ', '')
# 1. Researcher Boost
# Check if any keyword matches the researcher name significantly
for kw in search_keywords:
if len(kw) < 2: continue
if fuzz.partial_ratio(kw.lower(), item_researcher.lower()) >= SCORING.fuzzy_researcher_threshold:
score += SCORING.researcher_boost # Boost for researcher match
break
# 2. Methodology Boost
for kw in search_keywords:
if len(kw) < 2: continue
if fuzz.partial_ratio(kw.lower(), item_method.lower()) >= SCORING.fuzzy_method_threshold:
score += SCORING.methodology_boost # Boost for methodology match
break
# 3. Title Specific Boost (NEW)
item_title = meta.get('λ¦¬μ„œμΉ˜ 제λͺ©', '')
for kw in search_keywords:
if len(kw) < 2: continue
# [Stop Word Check] Skip boost for generic terms like "λ¦¬μ„œμΉ˜"
if kw in STOP_WORDS: continue
# [Title Match Bonus]
# If a keyword matches the TITLE significantly, it's likely very relevant.
if fuzz.partial_ratio(kw.lower(), item_title.lower()) >= SCORING.fuzzy_title_threshold:
score += SCORING.title_boost # Strong boost for title match
break
# Methodology Boosting (Legacy Filter Support)
if filters:
if filters.get('methodology'):
target_method = filters['methodology']
if fuzz.partial_ratio(target_method.lower(), item_method.lower()) >= 80:
score += SCORING.filter_force_boost # Force boost to top
# Researcher Boosting
if filters.get('researcher'):
target_researcher = filters['researcher']
# Fuzzy match for names (e.g. "Yondu" vs "Yondu Kim")
if fuzz.partial_ratio(target_researcher.lower(), item_researcher.lower()) >= SCORING.min_score:
score += SCORING.researcher_boost # Significant boost
# Date Boosting
if filters.get('date'):
target_date = filters['date'] # Expected YYYY
if target_date in item_date:
score += 200
if score >= min_score: # Use dynamic threshold (Default 80)
scored_item = dict(item)
scored_item['_score'] = score
meta_results.append(scored_item)
meta_results.sort(key=lambda x: x.get('_score', 0), reverse=True)
if k is None: k = 100
k = int(k)
top_meta = meta_results[:k]
final_results = []
seen = set()
def _build_internal_result(item, match_type):
meta = item.get('metadata', {})
detail = item.get('상세 정보', {})
related = detail.get('κ΄€λ ¨ 자료', [])
links_str = ""
if related:
links_str = "\nRelated Materials:\n" + "\n".join([f"- {l.get('제λͺ©')}: {l.get('url')}" for l in related])
return {
'source': 'INTERNAL',
'type': match_type,
'id': item.get('id'),
'title': meta.get('λ¦¬μ„œμΉ˜ 제λͺ©'),
'date': meta.get('λ‚ μ§œ', 'N/A'),
'researcher': meta.get('λ‹΄λ‹Ή λ¦¬μ„œμ²˜', 'N/A'),
'methodology': meta.get('방법둠', 'N/A'),
'domain': meta.get('도메인', 'N/A'),
'content': detail.get('μ£Όμš” λ‚΄μš© μš”μ•½'),
'link': f"{meta.get('결과 링크')}{links_str}",
'_score': item.get('_score', 0)
}
# High Confidence
for item in top_meta:
if item.get('_score', 0) >= SCORING.high_confidence_threshold:
rid = item.get('id')
if rid in seen: continue
final_results.append(_build_internal_result(item, 'Metadata Match (High Confidence)'))
seen.add(rid)
# Medium Confidence (remainder)
for item in top_meta:
rid = item.get('id')
if rid in seen: continue
final_results.append(_build_internal_result(item, 'Metadata Match (Medium Confidence)'))
seen.add(rid)
# 3. Global Sort by Score (Primary) and Date (Secondary)
def sort_key(item):
# Primary: Score (DESC)
score = item.get('_score', 0)
# Secondary: Date (DESC)
d_str = item.get('date', '')
clean_d = re.sub(r'[^0-9]', '', str(d_str))
date_val = 0
try:
if clean_d: date_val = int(clean_d)
except ValueError:
pass
return (score, date_val)
final_results.sort(key=sort_key, reverse=True)
return final_results
def search_baymard(self, keywords: List[str], k=5, min_score: int = 80) -> List[Dict]:
"""Search Baymard Guidelines"""
return self._search_external(
source_label='BAYMARD', data=self.baymard_data,
keywords=keywords,
text_fields=['title', 'description', 'issue'],
title_field='title', content_field='advice', link_field='url',
k=k, min_score=min_score,
)
def search_nng(self, keywords: List[str], k=5, min_score: int = 80) -> List[Dict]:
"""Search NN/g Articles"""
return self._search_external(
source_label='NNG', data=self.nng_data,
keywords=keywords,
text_fields=['title', 'description'],
title_field='title', content_field='description', link_field='url',
k=k, min_score=min_score,
date_field='date',
)
# ──────────────────────────────────────────────
# Hybrid Search (Fuzzy + Semantic via RRF)
# ──────────────────────────────────────────────
def hybrid_search(
self,
keywords: List[str],
query: str,
source: Optional[str] = None,
k: int = 10,
min_score: int = 60,
) -> List[Dict]:
"""Combine fuzzy and semantic search results using Reciprocal Rank Fusion.
Args:
keywords: Keywords for fuzzy search.
query: Natural language query for semantic search.
source: Optional source filter ("INTERNAL", "BAYMARD", "NNG").
k: Max results to return.
min_score: Minimum fuzzy score threshold.
Returns:
Merged results sorted by RRF score, each dict includes 'match_type'.
"""
rrf_k = SCORING.rrf_k
sources_to_search = [source.upper()] if source else ["INTERNAL", "BAYMARD", "NNG"]
# 1. Collect fuzzy results
fuzzy_results = []
for src in sources_to_search:
if src == "INTERNAL":
fuzzy_results.extend(self.search_internal(keywords, k=k, min_score=min_score))
elif src == "BAYMARD":
fuzzy_results.extend(self.search_baymard(keywords, k=k, min_score=min_score))
elif src == "NNG":
fuzzy_results.extend(self.search_nng(keywords, k=k, min_score=min_score))
# 2. Collect semantic results
semantic_hits = []
if self.vector_store and query:
semantic_hits = self.vector_store.semantic_search(
query=query, k=k, source_filter=source
)
# If no semantic results, return fuzzy results as-is
if not semantic_hits:
for r in fuzzy_results:
r["match_type"] = "Fuzzy Match"
return fuzzy_results[:k]
# 3. Build rank maps (doc_id β†’ rank, 0-indexed)
fuzzy_rank = {}
for rank, item in enumerate(fuzzy_results):
doc_id = str(item.get("id", ""))
src = item.get("source", "")
key = f"{src}:{doc_id}"
if key not in fuzzy_rank:
fuzzy_rank[key] = rank
semantic_rank = {}
for rank, hit in enumerate(semantic_hits):
key = f"{hit['source']}:{hit['doc_id']}"
if key not in semantic_rank:
semantic_rank[key] = rank
# 4. Compute RRF scores
all_keys = set(fuzzy_rank.keys()) | set(semantic_rank.keys())
rrf_scores = {}
for key in all_keys:
score = 0.0
if key in fuzzy_rank:
score += 1.0 / (rrf_k + fuzzy_rank[key])
if key in semantic_rank:
score += 1.0 / (rrf_k + semantic_rank[key])
rrf_scores[key] = score
# 5. Build result list
# Map fuzzy results by key for quick lookup
fuzzy_map = {}
for item in fuzzy_results:
key = f"{item.get('source', '')}:{item.get('id', '')}"
if key not in fuzzy_map:
fuzzy_map[key] = item
sorted_keys = sorted(rrf_scores.keys(), key=lambda k: rrf_scores[k], reverse=True)
merged = []
for key in sorted_keys[:k]:
in_fuzzy = key in fuzzy_rank
in_semantic = key in semantic_rank
if in_fuzzy and in_semantic:
match_type = "Hybrid Match"
elif in_fuzzy:
match_type = "Fuzzy Match"
else:
match_type = "Semantic Match"
if key in fuzzy_map:
result = dict(fuzzy_map[key])
result["match_type"] = match_type
merged.append(result)
else:
# Semantic-only hit: build result from document data
result = self._build_result_from_semantic(key)
if result:
result["match_type"] = match_type
merged.append(result)
return merged
def _build_result_from_semantic(self, key: str) -> Optional[Dict]:
"""Build a search result dict for a semantic-only hit."""
parts = key.split(":", 1)
if len(parts) != 2:
return None
source, doc_id = parts
doc = self.get_document_by_id(doc_id, source)
if not doc:
return None
if source == "INTERNAL":
return {
"source": "INTERNAL",
"type": "Semantic Match",
"id": doc_id,
"title": doc.get("title", ""),
"date": doc.get("date", ""),
"researcher": doc.get("researcher", ""),
"methodology": doc.get("methodology", ""),
"domain": doc.get("domain", ""),
"content": doc.get("summary", ""),
"link": doc.get("link", ""),
}
elif source == "BAYMARD":
return {
"source": "BAYMARD",
"type": "Semantic Match",
"id": doc_id,
"title": doc.get("title", ""),
"content": doc.get("advice", "") or doc.get("full_content", ""),
"link": doc.get("link", ""),
}
elif source == "NNG":
return {
"source": "NNG",
"type": "Semantic Match",
"id": doc_id,
"title": doc.get("title", ""),
"content": doc.get("description", ""),
"link": doc.get("link", ""),
"date": doc.get("date", ""),
}
return None
def get_specific_item(self, query: str) -> Dict[str, Any]:
"""
Finds a specific item (Internal/Baymard/NNG) best matching the query (Title/ID/Link).
Returns Dict with 'content' (FULL), 'title', 'source', etc. or None if not found.
"""
# 1. Normalize query
query_norm = query.strip().lower()
best_match = None
best_score = 0
# Helper to check match
def check_match(item, source_type):
nonlocal best_match, best_score
# Fields to check
rid = str(item.get('id', ''))
# Handle different data structures
if source_type == 'INTERNAL':
meta = item.get('metadata', {})
title = meta.get('λ¦¬μ„œμΉ˜ 제λͺ©', '')
link = meta.get('결과 링크', '')
content = item.get('상세 정보', {}).get('μ£Όμš” λ‚΄μš© μš”μ•½', '') # Will enrich later
elif source_type == 'BAYMARD':
title = item.get('title', '')
link = item.get('url', '')
content = item.get('advice', '')
elif source_type == 'NNG':
title = item.get('title', '')
link = item.get('url', '')
content = item.get('description', '')
# Calculate Score
# Exact ID Match
if rid.lower() in query_norm:
score = 100
# URL Match
elif query_norm in link.lower() and len(query_norm) > 10:
score = 100
# Title Fuzzy Match
else:
score = fuzz.ratio(query_norm, title.lower())
# Partial match for long titles if query is significant part
if len(query_norm) > 5:
partial = fuzz.partial_ratio(query_norm, title.lower())
if partial > score: score = partial
if score > 85 and score > best_score:
best_score = score
best_match = {
"source": source_type,
"id": rid,
"title": title,
"link": link,
"content": content, # Placeholder, will fetch full doc if needed
"original_item": item
}
# Scan Data sources
for item in self.internal_data: check_match(item, 'INTERNAL')
for item in self.baymard_data: check_match(item, 'BAYMARD')
for item in self.nng_data: check_match(item, 'NNG')
if best_match:
logger.info(f"Specific Item Match: {best_match['title']} ({best_match['source']}) - Score: {best_score}")
return best_match
return None
# ──────────────────────────────────────────────
# Agentic Retriever Helper Methods
# ──────────────────────────────────────────────
def _read_md_file(self, file_path: str) -> str:
"""Read a markdown file from DATA_DIR. Returns content or empty string."""
try:
full_path = os.path.join(str(DATA_DIR), file_path.lstrip('/'))
if os.path.exists(full_path):
with open(full_path, 'r', encoding='utf-8') as f:
return f.read()
except Exception as e:
logger.warning(f"Failed to read MD file {file_path}: {e}")
return ""
def get_document_by_id(self, doc_id: str, source: str) -> Optional[Dict[str, Any]]:
"""Returns full document content by ID and source. Used by agentic retriever tools."""
source = source.upper()
if source == "INTERNAL":
item = next((d for d in self.internal_data if d.get('id') == doc_id), None)
if not item:
return None
meta = item.get('metadata', {})
detail = item.get('상세 정보', {})
# Read full MD content if available
file_path = meta.get('file_path', '')
md_content = self._read_md_file(file_path) if file_path and file_path != "파일 μ—†μŒ" else ""
related = detail.get('κ΄€λ ¨ 자료', [])
links_str = ""
if related:
links_str = "\nκ΄€λ ¨ 자료:\n" + "\n".join([f"- {l.get('제λͺ©')}: {l.get('url')}" for l in related])
return {
'id': doc_id,
'source': 'INTERNAL',
'title': meta.get('λ¦¬μ„œμΉ˜ 제λͺ©', ''),
'date': meta.get('λ‚ μ§œ', ''),
'researcher': meta.get('λ‹΄λ‹Ή λ¦¬μ„œμ²˜', ''),
'methodology': meta.get('방법둠', ''),
'domain': meta.get('도메인', ''),
'tags': meta.get('νƒœκ·Έ', ''),
'summary': detail.get('μ£Όμš” λ‚΄μš© μš”μ•½', ''),
'full_content': md_content or detail.get('μ£Όμš” λ‚΄μš© μš”μ•½', ''),
'link': meta.get('결과 링크', '') + links_str,
}
elif source == "BAYMARD":
item = next((d for d in self.baymard_data if str(d.get('id')) == str(doc_id)), None)
if not item:
return None
return {
'id': str(item.get('id')),
'source': 'BAYMARD',
'title': item.get('title', ''),
'topic': item.get('topic', ''),
'issue': item.get('issue', ''),
'advice': item.get('advice', ''),
'description': item.get('description', ''),
'full_content': f"Issue:\n{item.get('issue', '')}\n\nAdvice:\n{item.get('advice', '')}\n\nDescription:\n{item.get('description', '')}",
'link': item.get('url', ''),
}
elif source == "NNG":
item = next((d for d in self.nng_data if d.get('id') == doc_id), None)
if not item:
return None
contents_list = item.get('contents', [])
contents_str = "\n".join(f"- {c}" for c in contents_list) if isinstance(contents_list, list) else str(contents_list)
return {
'id': doc_id,
'source': 'NNG',
'title': item.get('title', ''),
'date': item.get('date', ''),
'authors': item.get('authors', []),
'topics': item.get('topics', []),
'description': item.get('description', ''),
'contents_outline': contents_str,
'full_content': f"Description:\n{item.get('description', '')}\n\nContents:\n{contents_str}",
'link': item.get('url', ''),
}
return None
def list_topics(self, source: str) -> List[Dict[str, Any]]:
"""Returns list of topics with document counts for a given source."""
source = source.upper()
if source == "BAYMARD":
counter = Counter(item.get('topic', 'Unknown') for item in self.baymard_data)
return [{'topic': topic, 'count': count} for topic, count in counter.most_common()]
elif source == "NNG":
counter: Counter = Counter()
for item in self.nng_data:
for t in item.get('topics', []):
counter[t] += 1
return [{'topic': topic, 'count': count} for topic, count in counter.most_common()]
elif source == "INTERNAL":
counter = Counter(item.get('metadata', {}).get('도메인', 'Unknown') for item in self.internal_data)
return [{'topic': topic, 'count': count} for topic, count in counter.most_common()]
return []
def list_docs_by_topic(self, source: str, topic: str) -> List[Dict[str, Any]]:
"""Returns document ID/title/preview for a given source and topic."""
source = source.upper()
results = []
if source == "BAYMARD":
for item in self.baymard_data:
if item.get('topic', '').lower() == topic.lower():
results.append({
'id': str(item.get('id')),
'title': item.get('title', ''),
'preview': (item.get('description', '') or item.get('issue', ''))[:120],
})
elif source == "NNG":
for item in self.nng_data:
if topic.lower() in [t.lower() for t in item.get('topics', [])]:
results.append({
'id': item.get('id', ''),
'title': item.get('title', ''),
'preview': (item.get('description', ''))[:120],
})
elif source == "INTERNAL":
for item in self.internal_data:
if item.get('metadata', {}).get('도메인', '').lower() == topic.lower():
meta = item.get('metadata', {})
results.append({
'id': item.get('id', ''),
'title': meta.get('λ¦¬μ„œμΉ˜ 제λͺ©', ''),
'preview': (item.get('상세 정보', {}).get('μ£Όμš” λ‚΄μš© μš”μ•½', ''))[:120],
})
return results
# ──────────────────────────────────────────────
# Aggregation
# ──────────────────────────────────────────────
_INTERNAL_GROUP_FIELDS = {
"year": lambda item: item.get('metadata', {}).get('λ‚ μ§œ', '')[:4] or 'Unknown',
"domain": lambda item: item.get('metadata', {}).get('도메인', '') or 'Unknown',
"methodology": lambda item: item.get('metadata', {}).get('방법둠', '') or 'Unknown',
"researcher": lambda item: item.get('metadata', {}).get('λ‹΄λ‹Ή λ¦¬μ„œμ²˜', '') or 'Unknown',
"tag": lambda item: item.get('metadata', {}).get('νƒœκ·Έ', '') or 'Unknown',
}
_BAYMARD_GROUP_FIELDS = {
"topic": lambda item: [item.get('topic', 'Unknown')],
"theme": lambda item: [item.get('theme', 'Unknown')],
"platform": lambda item: item.get('platforms', []) or ['Unknown'],
"importance": lambda item: [str(item.get('importance', 'Unknown'))],
}
_NNG_GROUP_FIELDS = {
"topic": lambda item: item.get('topics', []) or ['Unknown'],
"author": lambda item: item.get('authors', []) or ['Unknown'],
"year": lambda item: [Retriever._extract_year_from_text(item.get('date', ''))],
}
@staticmethod
def _extract_year_from_text(date_text: str) -> str:
"""Extract a 4-digit year from free-form date text like 'June 25, 2023'."""
if not date_text:
return 'Unknown'
m = re.search(r'\b(19|20)\d{2}\b', date_text)
return m.group(0) if m else 'Unknown'
def aggregate(self, source: str, group_by: str,
filter_field: str = None, filter_value: str = None) -> Dict:
"""Aggregate document counts by a metadata field, with optional filtering.
Args:
source: "INTERNAL", "BAYMARD", or "NNG".
group_by: Field name to group by (source-specific).
filter_field: Optional field to filter on before aggregating.
filter_value: Value for filter (case-insensitive partial match).
Returns:
Dict with source, group_by, total_docs, counts (list of (value, count) tuples),
and optional filter info.
"""
source = source.upper()
# ALL: aggregate across all sources, grouped by source name
if source == "ALL":
counts = [
("INTERNAL", len(self.internal_data)),
("BAYMARD", len(self.baymard_data)),
("NNG", len(self.nng_data)),
]
total = sum(c for _, c in counts)
return {
"source": "ALL",
"group_by": "source",
"total_docs": total,
"counts": counts,
"filter": None,
}
# Select data and field maps
if source == "INTERNAL":
data = self.internal_data
group_map = self._INTERNAL_GROUP_FIELDS
filter_map = self._INTERNAL_GROUP_FIELDS
elif source == "BAYMARD":
data = self.baymard_data
group_map = self._BAYMARD_GROUP_FIELDS
filter_map = self._BAYMARD_GROUP_FIELDS
elif source == "NNG":
data = self.nng_data
group_map = self._NNG_GROUP_FIELDS
filter_map = self._NNG_GROUP_FIELDS
else:
return {"error": f"Unknown source: {source}. Use INTERNAL, BAYMARD, NNG, or ALL."}
if group_by not in group_map:
valid = ", ".join(group_map.keys())
return {"error": f"Invalid group_by '{group_by}' for {source}. Valid: {valid}"}
# Validate filter_field
if filter_field and filter_field not in filter_map:
valid = ", ".join(filter_map.keys())
return {"error": f"Invalid filter_field '{filter_field}' for {source}. Valid: {valid}"}
group_fn = group_map[group_by]
# Apply optional filter
filtered_data = data
if filter_field and filter_value:
filter_fn = filter_map[filter_field]
fv_lower = filter_value.lower()
filtered_data = []
for item in data:
values = filter_fn(item)
if isinstance(values, str):
values = [values]
if any(fv_lower in v.lower() for v in values):
filtered_data.append(item)
# Count
counter: Counter = Counter()
for item in filtered_data:
values = group_fn(item)
if isinstance(values, str):
values = [values]
for v in values:
counter[v] += 1
return {
"source": source,
"group_by": group_by,
"total_docs": len(filtered_data),
"counts": counter.most_common(),
"filter": {"field": filter_field, "value": filter_value} if filter_field else None,
}
def filter_by_metadata(self, source: str, field: str, value: str,
max_results: int = 20) -> List[Dict]:
"""Filter documents by metadata field and return matching documents with previews.
Args:
source: "INTERNAL", "BAYMARD", or "NNG".
field: Metadata field to filter on (same as aggregate group_by fields).
value: Filter value (case-insensitive partial match).
max_results: Maximum documents to return.
Returns:
List of matching documents with id, title, preview, and metadata.
"""
source = source.upper()
field_maps = {
"INTERNAL": (self.internal_data, self._INTERNAL_GROUP_FIELDS),
"BAYMARD": (self.baymard_data, self._BAYMARD_GROUP_FIELDS),
"NNG": (self.nng_data, self._NNG_GROUP_FIELDS),
}
if source not in field_maps:
return []
data, group_map = field_maps[source]
if field not in group_map:
return []
filter_fn = group_map[field]
val_lower = value.lower()
results = []
for item in data:
values = filter_fn(item)
if isinstance(values, str):
values = [values]
if not any(val_lower in v.lower() for v in values):
continue
if source == "INTERNAL":
meta = item.get('metadata', {})
detail = item.get('상세 정보', {})
results.append({
'id': item.get('id', ''),
'title': meta.get('λ¦¬μ„œμΉ˜ 제λͺ©', ''),
'source': 'INTERNAL',
'date': meta.get('λ‚ μ§œ', ''),
'domain': meta.get('도메인', ''),
'methodology': meta.get('방법둠', ''),
'researcher': meta.get('λ‹΄λ‹Ή λ¦¬μ„œμ²˜', ''),
'tags': meta.get('νƒœκ·Έ', ''),
'link': meta.get('결과 링크', ''),
'preview': (detail.get('μ£Όμš” λ‚΄μš© μš”μ•½', ''))[:250],
})
elif source == "BAYMARD":
results.append({
'id': item.get('id', ''),
'title': item.get('title', ''),
'source': 'BAYMARD',
'topic': item.get('topic', ''),
'importance': item.get('importance', ''),
'link': item.get('url', ''),
'preview': (item.get('description', ''))[:250],
})
elif source == "NNG":
results.append({
'id': item.get('id', ''),
'title': item.get('title', ''),
'source': 'NNG',
'authors': item.get('authors', []),
'topics': item.get('topics', []),
'link': item.get('url', ''),
'preview': (item.get('description', ''))[:250],
})
if len(results) >= max_results:
break
return results