slownews commited on
Commit
a8a41a8
ยท
verified ยท
1 Parent(s): 9456d31

Update streamlit_app.py

Browse files
Files changed (1) hide show
  1. streamlit_app.py +51 -241
streamlit_app.py CHANGED
@@ -1,248 +1,58 @@
1
- # ๋„คํŠธ์›Œํฌ ์—๋Ÿฌ ํ•ด๊ฒฐ๋œ ์Šฌ๋กœ๋ ˆํ„ฐ ๊ฒ€์ƒ‰ ์•ฑ
2
  import streamlit as st
3
- import pandas as pd
4
- import numpy as np
5
- from rank_bm25 import BM25Okapi
6
- import pickle
7
- import json
8
  import os
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
 
10
- # Hugging Face Spaces ์„ค์ •
11
- st.set_page_config(
12
- page_title="์Šฌ๋กœ๋ ˆํ„ฐ ์ง€๋Šฅํ˜• ๊ฒ€์ƒ‰",
13
- page_icon="๐Ÿ”",
14
- layout="wide"
15
- )
16
 
17
- # ๋ฐ์ดํ„ฐ ๊ฒฝ๋กœ ์ˆ˜์ • (๋ฃจํŠธ ๋””๋ ‰ํ† ๋ฆฌ)
18
- DATA_PATH = "."
19
 
20
- class HFSlowletterSearch:
21
- def __init__(self):
22
- self.embeddings = None
23
- self.bm25 = None
24
- self.processed_data = None
25
- self.metadata = {}
26
- self.system_loaded = False
27
-
28
- @st.cache_resource
29
- def load_system(_self):
30
- """์‹œ์Šคํ…œ ๋กœ๋“œ (HF Spaces ์ตœ์ ํ™”)"""
31
- try:
32
- # ํŒŒ์ผ ์กด์žฌ ํ™•์ธ
33
- required_files = [
34
- 'embeddings.npy',
35
- 'bm25_index.pkl',
36
- 'processed_data.pkl',
37
- 'system_metadata.json'
38
- ]
39
-
40
- missing_files = []
41
- for file in required_files:
42
- if not os.path.exists(os.path.join(DATA_PATH, file)):
43
- missing_files.append(file)
44
-
45
- if missing_files:
46
- st.error(f"ํ•„์š”ํ•œ ํŒŒ์ผ์ด ์—†์Šต๋‹ˆ๋‹ค: {missing_files}")
47
- return False
48
-
49
- # ์ž„๋ฒ ๋”ฉ ๋กœ๋“œ (์‚ฌ์šฉ์€ ์•ˆํ•˜์ง€๋งŒ ๋กœ๋“œ๋Š” ํ•จ)
50
- _self.embeddings = np.load(f"{DATA_PATH}/embeddings.npy")
51
-
52
- # BM25 ์ธ๋ฑ์Šค ๋กœ๋“œ
53
- with open(f"{DATA_PATH}/bm25_index.pkl", 'rb') as f:
54
- _self.bm25 = pickle.load(f)
55
-
56
- # ๋ฐ์ดํ„ฐ ๋กœ๋“œ
57
- with open(f"{DATA_PATH}/processed_data.pkl", 'rb') as f:
58
- _self.processed_data = pickle.load(f)
59
-
60
- # ๋ฉ”ํƒ€๋ฐ์ดํ„ฐ ๋กœ๋“œ
61
- with open(f"{DATA_PATH}/system_metadata.json", 'r') as f:
62
- _self.metadata = json.load(f)
63
-
64
- _self.system_loaded = True
65
- st.success("โœ… ์‹œ์Šคํ…œ ๋กœ๋“œ ์™„๋ฃŒ!")
66
- return True
67
-
68
- except Exception as e:
69
- st.error(f"์‹œ์Šคํ…œ ๋กœ๋“œ ์‹คํŒจ: {e}")
70
- return False
71
-
72
- def tokenize_korean(self, text):
73
- import re
74
- tokens = re.findall(r'\b\w+\b', str(text).lower())
75
- return [token for token in tokens if len(token) > 1]
76
-
77
- def search(self, query, top_k=10):
78
- """BM25 ๊ธฐ๋ฐ˜ ๊ฒ€์ƒ‰ (๋„คํŠธ์›Œํฌ ํ˜ธ์ถœ ์—†์Œ)"""
79
- if not self.system_loaded:
80
- return []
81
-
82
- # BM25 ๊ฒ€์ƒ‰๋งŒ ์‚ฌ์šฉ (API ํ˜ธ์ถœ ์ œ๊ฑฐ)
83
- tokenized_query = self.tokenize_korean(query)
84
- bm25_scores = self.bm25.get_scores(tokenized_query)
85
-
86
- # ์ƒ์œ„ ๊ฒฐ๊ณผ ์„ ํƒ
87
- top_indices = np.argsort(bm25_scores)[::-1][:top_k]
88
-
89
- results = []
90
- for rank, idx in enumerate(top_indices, 1):
91
- if bm25_scores[idx] > 0: # ๊ด€๋ จ์„ฑ ์žˆ๋Š” ๊ฒฐ๊ณผ๋งŒ
92
- row = self.processed_data.iloc[idx]
93
- results.append({
94
- 'rank': rank,
95
- 'title': row.get('h3_title', ''),
96
- 'date': row.get('date', ''),
97
- 'newsletter': row.get('h1_title', ''),
98
- 'entities': row.get('entities', ''),
99
- 'events': row.get('events', ''),
100
- 'content': row.get('h3_content_text', ''),
101
- 'score': float(bm25_scores[idx])
102
- })
103
-
104
- return results
105
 
106
- def main():
107
- # ์ œ๋ชฉ ๋ฐ ์„ค๋ช…
108
- st.title("๐Ÿ” ์Šฌ๋กœ๋ ˆํ„ฐ ์ง€๋Šฅํ˜• ๊ฒ€์ƒ‰")
109
- st.markdown("**2๋…„๊ฐ„์˜ ์Šฌ๋กœ๋ ˆํ„ฐ ์•„์นด์ด๋ธŒ๋ฅผ ๊ฒ€์ƒ‰ํ•˜์„ธ์š”!**")
110
- st.info("๐Ÿ’ก ํ‚ค์›Œ๋“œ ๊ธฐ๋ฐ˜ BM25 ๊ฒ€์ƒ‰์„ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค (API ํ‚ค ๋ถˆํ•„์š”)")
111
-
112
- # ์‹œ์Šคํ…œ ์ดˆ๊ธฐํ™”
113
- search_system = HFSlowletterSearch()
114
-
115
- # ์‹œ์Šคํ…œ ๋กœ๋“œ
116
- with st.spinner("๐Ÿ”„ ์‹œ์Šคํ…œ ๋กœ๋”ฉ ์ค‘..."):
117
- system_loaded = search_system.load_system()
118
-
119
- if system_loaded:
120
- # ์‹œ์Šคํ…œ ์ •๋ณด
121
- col1, col2, col3 = st.columns(3)
122
- with col1:
123
- st.metric("๐Ÿ“Š ์ด ๋ฌธ์„œ", f"{search_system.metadata.get('total_documents', 0):,}๊ฐœ")
124
- with col2:
125
- st.metric("๐Ÿ“… ์ƒ์„ฑ์ผ", search_system.metadata.get('created_at', 'Unknown')[:10])
126
- with col3:
127
- st.metric("๐Ÿ”„ ์—…๋ฐ์ดํŠธ", search_system.metadata.get('last_updated', 'Unknown')[:10])
128
-
129
- st.divider()
130
-
131
- # ๋น ๋ฅธ ๊ฒ€์ƒ‰ ๋ฒ„ํŠผ
132
- st.subheader("๐Ÿš€ ๋น ๋ฅธ ๊ฒ€์ƒ‰")
133
-
134
- quick_queries = [
135
- "์œค์„์—ด ๋Œ€ํ†ต๋ น",
136
- "์ด์žฌ๋ช… ๋Œ€ํ†ต๋ น",
137
- "๋ช…ํƒœ๊ท  ์‚ฌ๊ฑด",
138
- "ํƒ„ํ•ต",
139
- "๊ฒฝ์ œ ์ •์ฑ…",
140
- "๋ถํ•œ"
141
- ]
142
-
143
- cols = st.columns(3)
144
- selected_query = None
145
-
146
- for i, query in enumerate(quick_queries):
147
- col = cols[i % 3]
148
- if col.button(query, key=f"quick_{i}"):
149
- selected_query = query
150
-
151
- # ๊ฒ€์ƒ‰์ฐฝ
152
- search_query = st.text_input(
153
- "๊ฒ€์ƒ‰์–ด๋ฅผ ์ž…๋ ฅํ•˜์„ธ์š”:",
154
- value=selected_query or "",
155
- placeholder="์˜ˆ: ์œค์„์—ด ๋Œ€ํ†ต๋ น ์ตœ๊ทผ ๋…ผ๋ž€"
156
- )
157
-
158
- # ๊ฒ€์ƒ‰ ์„ค์ •
159
- col1, col2 = st.columns([3, 1])
160
- with col2:
161
- top_k = st.selectbox("๊ฒฐ๊ณผ ๊ฐœ์ˆ˜", [5, 10, 20, 30], index=1)
162
-
163
- # ๊ฒ€์ƒ‰ ์‹คํ–‰
164
- if search_query:
165
- with st.spinner("๐Ÿ” ๊ฒ€์ƒ‰ ์ค‘..."):
166
- results = search_system.search(search_query, top_k=top_k)
167
-
168
- if results:
169
- st.success(f"๐ŸŽ‰ {len(results)}๊ฐœ ๊ฒฐ๊ณผ๋ฅผ ์ฐพ์•˜์Šต๋‹ˆ๋‹ค!")
170
-
171
- # ๊ฒฐ๊ณผ ํ‘œ์‹œ
172
- for result in results:
173
- with st.expander(
174
- f"๐Ÿ† {result['rank']}. {result['title']} ({result['date']})",
175
- expanded=result['rank'] <= 3
176
- ):
177
- col1, col2 = st.columns([3, 1])
178
-
179
- with col1:
180
- if result['entities']:
181
- entities = result['entities'].split(';')[:5]
182
- entities = [e.strip() for e in entities if e.strip()]
183
- if entities:
184
- st.write("**๐Ÿ‘ฅ ๊ด€๋ จ ์ธ๋ฌผ/๊ธฐ๊ด€:**", ", ".join(entities))
185
-
186
- if result['events']:
187
- events = result['events'].split(';')[:3]
188
- events = [e.strip() for e in events if e.strip()]
189
- if events:
190
- st.write("**๐Ÿ“‹ ์ฃผ์š” ์‚ฌ๊ฑด:**", ", ".join(events))
191
-
192
- if result['content']:
193
- st.write("**๐Ÿ“ ๋‚ด์šฉ:**")
194
- content = result['content']
195
- if len(content) > 300:
196
- st.write(content[:300] + "...")
197
- else:
198
- st.write(content)
199
-
200
- with col2:
201
- st.metric("๊ฒ€์ƒ‰ ์ ์ˆ˜", f"{result['score']:.2f}")
202
- if result['newsletter']:
203
- st.write(f"**๐Ÿ“„** {result['newsletter']}")
204
- else:
205
- st.warning("๐Ÿ” ๊ฒ€์ƒ‰ ๊ฒฐ๊ณผ๊ฐ€ ์—†์Šต๋‹ˆ๋‹ค. ๋‹ค๋ฅธ ํ‚ค์›Œ๋“œ๋กœ ์‹œ๋„ํ•ด๋ณด์„ธ์š”.")
206
- st.info("๐Ÿ’ก ํŒ: ๋” ๊ฐ„๋‹จํ•œ ํ‚ค์›Œ๋“œ๋กœ ๊ฒ€์ƒ‰ํ•ด๋ณด์„ธ์š” (์˜ˆ: '์œค์„์—ด', '์ด์žฌ๋ช…', 'ํƒ„ํ•ต')")
207
-
208
- # ์‚ฌ์šฉ๋ฒ• ์•ˆ๋‚ด
209
- with st.expander("๐Ÿ’ก ์‚ฌ์šฉ๋ฒ• ์•ˆ๋‚ด"):
210
- st.markdown("""
211
- ### ๐Ÿ” ๊ฒ€์ƒ‰ ํŒ
212
- - **์ธ๋ฌผ๋ช…**: "์œค์„์—ด", "์ด์žฌ๋ช…", "๋ช…ํƒœ๊ท "
213
- - **์‚ฌ๊ฑด๋ช…**: "ํƒ„ํ•ต", "ํŠน๊ฒ€", "๋“œ๋ก "
214
- - **์ฃผ์ œ**: "๊ฒฝ์ œ ์ •์ฑ…", "๋ถํ•œ ๊ด€๋ จ"
215
- - **๋ณตํ•ฉ ๊ฒ€์ƒ‰**: "์œค์„์—ด ํƒ„ํ•ต ๊ฐ€๋Šฅ์„ฑ"
216
-
217
- ### โšก ์‹œ์Šคํ…œ ์ •๋ณด
218
- - **๊ฒ€์ƒ‰ ๋ฐฉ์‹**: BM25 ํ‚ค์›Œ๋“œ ๋งค์นญ
219
- - **๋ฐ์ดํ„ฐ ๋ฒ”์œ„**: 2023๋…„ 4์›” ~ 2025๋…„ 7์›”
220
- - **์ด ๋ฌธ์„œ ์ˆ˜**: 13,000+ ๊ฐœ ์Šฌ๋กœ๋ ˆํ„ฐ ๏ฟฝ๏ฟฝ๏ฟฝ์‚ฌ
221
- - **์—…๋ฐ์ดํŠธ**: ์ •๊ธฐ์ ์œผ๋กœ ์ƒˆ๋กœ์šด ๊ธฐ์‚ฌ ์ถ”๊ฐ€
222
-
223
- ### ๐ŸŽฏ ๊ธฐ๋Šฅ
224
- - ๐Ÿ” **์ •ํ™•ํ•œ ํ‚ค์›Œ๋“œ ๋งค์นญ**: ์ž…๋ ฅํ•œ ๋‹จ์–ด์™€ ์ •ํ™•ํžˆ ์ผ์น˜ํ•˜๋Š” ๊ธฐ์‚ฌ ์šฐ์„ 
225
- - ๐Ÿ“Š **๊ด€๋ จ๋„ ์ˆœ ์ •๋ ฌ**: ๊ฐ€์žฅ ๊ด€๋ จ์„ฑ ๋†’์€ ๊ฒฐ๊ณผ ๋จผ์ € ํ‘œ์‹œ
226
- - ๐Ÿ“… **์‹œ๊ฐ„์ˆœ ์ถ”์ **: ์‚ฌ๊ฑด์˜ ์‹œ๊ฐ„์  ํ๋ฆ„ ํŒŒ์•… ๊ฐ€๋Šฅ
227
- - ๐Ÿ‘ฅ **์ธ๋ฌผ/์‚ฌ๊ฑด ์—ฐ๊ฒฐ**: ๊ด€๋ จ ์ธ๋ฌผ๊ณผ ์‚ฌ๊ฑด ์ž๋™ ํ‘œ์‹œ
228
-
229
- ---
230
- ๐Ÿ’ป **๊ธฐ์ˆ  ์Šคํƒ**: BM25 + Contextual Retrieval | ๐Ÿ—๏ธ **ํ”Œ๋žซํผ**: Hugging Face Spaces
231
- """)
232
-
233
- else:
234
- st.error("โŒ ์‹œ์Šคํ…œ์„ ๋กœ๋“œํ•  ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค.")
235
- st.info("๐Ÿ”ง ํ•„์š”ํ•œ ๋ฐ์ดํ„ฐ ํŒŒ์ผ๋“ค์ด ์—…๋กœ๋“œ๋˜์ง€ ์•Š์•˜์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค.")
236
-
237
- # ๋””๋ฒ„๊น… ์ •๋ณด
238
- with st.expander("๐Ÿ” ๋””๋ฒ„๊น… ์ •๋ณด"):
239
- st.write("ํ•„์š”ํ•œ ํŒŒ์ผ๋“ค:")
240
- files_status = {}
241
- for file in ['embeddings.npy', 'bm25_index.pkl', 'processed_data.pkl', 'system_metadata.json']:
242
- exists = os.path.exists(file)
243
- files_status[file] = "โœ…" if exists else "โŒ"
244
-
245
- st.json(files_status)
246
 
247
- if __name__ == "__main__":
248
- main()
 
 
1
+ # ์ตœ์†Œ ํ…Œ์ŠคํŠธ ์•ฑ - ๋„คํŠธ์›Œํฌ ์—๋Ÿฌ ๋””๋ฒ„๊น…์šฉ
2
  import streamlit as st
 
 
 
 
 
3
  import os
4
+ import json
5
+
6
+ st.title("๐Ÿ” ์Šฌ๋กœ๋ ˆํ„ฐ ๊ฒ€์ƒ‰ - ๋””๋ฒ„๊น…")
7
+
8
+ st.write("## ๐Ÿ“ ํŒŒ์ผ ํ™•์ธ")
9
+
10
+ # ํ˜„์žฌ ๋””๋ ‰ํ† ๋ฆฌ ํŒŒ์ผ ๋ชฉ๋ก
11
+ files = os.listdir(".")
12
+ st.write("ํ˜„์žฌ ๋””๋ ‰ํ† ๋ฆฌ ํŒŒ์ผ๋“ค:")
13
+ for file in files:
14
+ size = os.path.getsize(file) if os.path.isfile(file) else "ํด๋”"
15
+ st.write(f"- {file}: {size}")
16
+
17
+ st.write("## ๐Ÿ” ์‹œ์Šคํ…œ ํŒŒ์ผ ํ…Œ์ŠคํŠธ")
18
+
19
+ # ๊ฐ ํŒŒ์ผ ๊ฐœ๋ณ„ ํ…Œ์ŠคํŠธ
20
+ test_results = {}
21
+
22
+ # 1. ๋ฉ”ํƒ€๋ฐ์ดํ„ฐ ํŒŒ์ผ ํ…Œ์ŠคํŠธ
23
+ try:
24
+ with open("system_metadata.json", 'r') as f:
25
+ metadata = json.load(f)
26
+ test_results["system_metadata.json"] = f"โœ… ์„ฑ๊ณต - {len(metadata)} ๊ฐœ ํ•ญ๋ชฉ"
27
+ st.json(metadata)
28
+ except Exception as e:
29
+ test_results["system_metadata.json"] = f"โŒ ์‹คํŒจ: {e}"
30
+
31
+ # 2. ๋‹ค๋ฅธ ํŒŒ์ผ๋“ค ํฌ๊ธฐ๋งŒ ํ™•์ธ
32
+ for file in ["embeddings.npy", "bm25_index.pkl", "processed_data.pkl"]:
33
+ try:
34
+ size = os.path.getsize(file)
35
+ test_results[file] = f"โœ… ํŒŒ์ผ ์กด์žฌ - {size/1024/1024:.1f}MB"
36
+ except Exception as e:
37
+ test_results[file] = f"โŒ ํŒŒ์ผ ์—†์Œ: {e}"
38
+
39
+ st.write("## ๐Ÿ“Š ํ…Œ์ŠคํŠธ ๊ฒฐ๊ณผ")
40
+ for file, result in test_results.items():
41
+ st.write(f"**{file}**: {result}")
42
 
43
+ # ๊ฐ„๋‹จํ•œ ๊ฒ€์ƒ‰ ์ธํ„ฐํŽ˜์ด์Šค (ํŒŒ์ผ ๋กœ๋”ฉ ์—†์ด)
44
+ st.write("## ๐Ÿงช ๊ธฐ๋ณธ ๊ธฐ๋Šฅ ํ…Œ์ŠคํŠธ")
 
 
 
 
45
 
46
+ search_query = st.text_input("ํ…Œ์ŠคํŠธ ๊ฒ€์ƒ‰์–ด๋ฅผ ์ž…๋ ฅํ•˜์„ธ์š”:")
 
47
 
48
+ if search_query:
49
+ st.success(f"์ž…๋ ฅ๋ฐ›์€ ๊ฒ€์ƒ‰์–ด: {search_query}")
50
+ st.info("ํŒŒ์ผ ๋กœ๋”ฉ์€ ๋‹ค์Œ ๋‹จ๊ณ„์—์„œ ํ…Œ์ŠคํŠธํ•ฉ๋‹ˆ๋‹ค.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
 
52
+ # ์‹œ์Šคํ…œ ์ •๋ณด
53
+ st.write("## โš™๏ธ ์‹œ์Šคํ…œ ์ •๋ณด")
54
+ st.write(f"Python ๋ฒ„์ „: {st.__version__}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
+ # ๋„คํŠธ์›Œํฌ ํ…Œ์ŠคํŠธ ์ œ๊ฑฐ
57
+ st.write("## โœ… ๋„คํŠธ์›Œํฌ ํ˜ธ์ถœ ์—†์Œ")
58
+ st.success("์ด ์•ฑ์€ ์–ด๋–ค ์™ธ๋ถ€ ๋„คํŠธ์›Œํฌ ํ˜ธ์ถœ๋„ ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค.")