| |
| import streamlit as st |
| import pandas as pd |
| import numpy as np |
| import pickle |
| import json |
| import os |
|
|
| st.set_page_config( |
| page_title="μ¬λ‘λ ν° μ§λ₯ν κ²μ", |
| page_icon="π", |
| layout="wide" |
| ) |
|
|
| class SafeSlowletterSearch: |
| def __init__(self): |
| self.bm25 = None |
| self.processed_data = None |
| self.metadata = {} |
| self.system_loaded = False |
| |
| def safe_load_json(self, filepath): |
| """JSON νμΌ μμ λ‘λ©""" |
| try: |
| with open(filepath, 'r', encoding='utf-8') as f: |
| return json.load(f) |
| except Exception as e: |
| st.error(f"JSON λ‘λ© μ€ν¨ ({filepath}): {e}") |
| return {} |
| |
| def safe_load_pickle(self, filepath): |
| """Pickle νμΌ μμ λ‘λ©""" |
| try: |
| with open(filepath, 'rb') as f: |
| return pickle.load(f) |
| except Exception as e: |
| st.error(f"Pickle λ‘λ© μ€ν¨ ({filepath}): {e}") |
| return None |
| |
| def safe_load_numpy(self, filepath): |
| """NumPy νμΌ μμ λ‘λ©""" |
| try: |
| return np.load(filepath, allow_pickle=True) |
| except Exception as e: |
| st.error(f"NumPy λ‘λ© μ€ν¨ ({filepath}): {e}") |
| return None |
| |
| def load_system_step_by_step(self): |
| """λ¨κ³λ³ μμ€ν
λ‘λ©""" |
| st.write("π μμ€ν
νμΌ λ‘λ© μμ...") |
| |
| |
| required_files = { |
| 'system_metadata.json': 'λ©νλ°μ΄ν°', |
| 'processed_data.pkl': 'μ²λ¦¬λ λ°μ΄ν°', |
| 'bm25_index.pkl': 'BM25 μΈλ±μ€', |
| 'embeddings.npy': 'μλ² λ© λ°μ΄ν°' |
| } |
| |
| missing_files = [] |
| for file, desc in required_files.items(): |
| if os.path.exists(file): |
| size = os.path.getsize(file) / 1024 / 1024 |
| st.success(f"β
{desc}: {file} ({size:.1f}MB)") |
| else: |
| st.error(f"β {desc}: {file} (μμ)") |
| missing_files.append(file) |
| |
| if missing_files: |
| st.error(f"νμ νμΌ λΆμ‘±: {missing_files}") |
| return False |
| |
| |
| st.write("π λ©νλ°μ΄ν° λ‘λ© μ€...") |
| self.metadata = self.safe_load_json('system_metadata.json') |
| if self.metadata: |
| st.success("β
λ©νλ°μ΄ν° λ‘λ© μλ£") |
| st.json(self.metadata) |
| |
| |
| st.write("π λ°μ΄ν° λ‘λ© μ€...") |
| self.processed_data = self.safe_load_pickle('processed_data.pkl') |
| if self.processed_data is not None: |
| st.success(f"β
λ°μ΄ν° λ‘λ© μλ£: {len(self.processed_data)}κ° ν") |
| else: |
| st.error("β λ°μ΄ν° λ‘λ© μ€ν¨") |
| return False |
| |
| |
| st.write("π BM25 μΈλ±μ€ λ‘λ© μ€...") |
| self.bm25 = self.safe_load_pickle('bm25_index.pkl') |
| if self.bm25 is not None: |
| st.success("β
BM25 μΈλ±μ€ λ‘λ© μλ£") |
| else: |
| st.error("β BM25 μΈλ±μ€ λ‘λ© μ€ν¨") |
| return False |
| |
| |
| st.info("βΉοΈ μλ² λ© νμΌμ μΌλ¨ μ€ν΅ (BM25λ§ μ¬μ©)") |
| |
| self.system_loaded = True |
| return True |
| |
| def tokenize_korean(self, text): |
| import re |
| tokens = re.findall(r'\b\w+\b', str(text).lower()) |
| return [token for token in tokens if len(token) > 1] |
| |
| def search(self, query, top_k=10): |
| """BM25 μ μ© κ²μ""" |
| if not self.system_loaded or self.bm25 is None: |
| return [] |
| |
| try: |
| tokenized_query = self.tokenize_korean(query) |
| bm25_scores = self.bm25.get_scores(tokenized_query) |
| |
| |
| top_indices = np.argsort(bm25_scores)[::-1][:top_k] |
| |
| results = [] |
| for rank, idx in enumerate(top_indices, 1): |
| if bm25_scores[idx] > 0: |
| row = self.processed_data.iloc[idx] |
| results.append({ |
| 'rank': rank, |
| 'title': row.get('h3_title', ''), |
| 'date': row.get('date', ''), |
| 'newsletter': row.get('h1_title', ''), |
| 'entities': row.get('entities', ''), |
| 'events': row.get('events', ''), |
| 'content': row.get('h3_content_text', ''), |
| 'score': float(bm25_scores[idx]) |
| }) |
| |
| return results |
| |
| except Exception as e: |
| st.error(f"κ²μ μ€ μ€λ₯: {e}") |
| return [] |
|
|
| def main(): |
| st.title("π μ¬λ‘λ ν° μ§λ₯ν κ²μ") |
| st.markdown("**Input Stream μλ¬ ν΄κ²° λ²μ **") |
| |
| |
| if 'search_system' not in st.session_state: |
| st.session_state.search_system = SafeSlowletterSearch() |
| st.session_state.system_loaded = False |
| |
| search_system = st.session_state.search_system |
| |
| |
| if not st.session_state.system_loaded: |
| with st.container(): |
| if st.button("π μμ€ν
λ‘λ© μμ", type="primary"): |
| success = search_system.load_system_step_by_step() |
| st.session_state.system_loaded = success |
| if success: |
| st.balloons() |
| |
| |
| if st.session_state.system_loaded: |
| st.divider() |
| st.subheader("π κ²μ") |
| |
| |
| quick_queries = ["μ€μμ΄", "μ΄μ¬λͺ
", "λͺ
νκ· ", "νν΅", "κ²½μ ", "λΆν"] |
| |
| cols = st.columns(3) |
| selected_query = None |
| |
| for i, query in enumerate(quick_queries): |
| col = cols[i % 3] |
| if col.button(query, key=f"quick_{i}"): |
| selected_query = query |
| |
| |
| search_query = st.text_input( |
| "κ²μμ΄:", |
| value=selected_query or "", |
| placeholder="μ: μ€μμ΄ λν΅λ Ή" |
| ) |
| |
| |
| if search_query: |
| with st.spinner("κ²μ μ€..."): |
| results = search_system.search(search_query, top_k=10) |
| |
| if results: |
| st.success(f"π {len(results)}κ° κ²°κ³Ό λ°κ²¬!") |
| |
| for result in results: |
| with st.expander(f"π {result['rank']}. {result['title']} ({result['date']})"): |
| col1, col2 = st.columns([3, 1]) |
| |
| with col1: |
| if result['entities']: |
| entities = [e.strip() for e in result['entities'].split(';')[:3] if e.strip()] |
| if entities: |
| st.write("**π₯ κ΄λ ¨ μΈλ¬Ό:**", ", ".join(entities)) |
| |
| if result['content']: |
| st.write("**π λ΄μ©:**") |
| st.write(result['content'][:200] + "..." if len(result['content']) > 200 else result['content']) |
| |
| with col2: |
| st.metric("μ μ", f"{result['score']:.2f}") |
| else: |
| st.warning("κ²μ κ²°κ³Όκ° μμ΅λλ€.") |
| |
| else: |
| st.info("π 'μμ€ν
λ‘λ© μμ' λ²νΌμ ν΄λ¦ν΄μ μμνμΈμ!") |
|
|
| if __name__ == "__main__": |
| main() |
