tjl8 commited on
Commit
2fd7752
·
verified ·
1 Parent(s): a5bb81a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +35 -179
app.py CHANGED
@@ -130,133 +130,6 @@
130
  # # summary = rag_summarize(collected, summarizer)
131
  # # st.success(summary)
132
 
133
- # import streamlit as st
134
- # import pandas as pd
135
- # import re
136
- # from sentence_transformers import SentenceTransformer
137
- # from transformers import pipeline
138
- # from sklearn.metrics.pairwise import cosine_similarity
139
- # from sklearn.feature_extraction.text import TfidfVectorizer
140
- # from datetime import datetime
141
-
142
- # # loading data
143
- # @st.cache_data
144
- # def load_data():
145
- # df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
146
- # df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
147
- # df = df.dropna(subset=['status_date'])
148
- # df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")
149
- # df["llama_insight"] = df["llama_insight"].fillna("")
150
- # df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
151
- # return df
152
-
153
- # @st.cache_resource
154
- # def load_models():
155
- # embed_model = SentenceTransformer('all-MiniLM-L6-v2')
156
- # summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
157
- # return embed_model, summarizer
158
-
159
- # @st.cache_data
160
- # def compute_embeddings(texts, _model):
161
- # return _model.encode(texts, show_progress_bar=True)
162
-
163
- # def semantic_search(query, embeddings, model, threshold=0.5):
164
- # query_embedding = model.encode([query])
165
- # sims = cosine_similarity(query_embedding, embeddings)[0]
166
- # return [(i, s) for i, s in enumerate(sims) if s > threshold]
167
-
168
-
169
- # def rag_summarize(texts, summarizer, top_k=5):
170
- # if not texts:
171
- # return "No relevant content to summarize."
172
- # vect = TfidfVectorizer()
173
- # m = vect.fit_transform(texts)
174
- # mean_vec = m.mean(axis=0).A
175
- # scores = cosine_similarity(mean_vec, m).flatten()
176
- # top_indices = scores.argsort()[::-1][:top_k]
177
- # ctx = "\n".join(texts[i] for i in top_indices)
178
- # prompt = "summarize: " + ctx[:1024]
179
- # out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
180
- # return out[0]['summary_text']
181
-
182
- # def extract_month_year(q):
183
- # month_map = {m: i for i, m in enumerate(
184
- # ["january", "february", "march", "april", "may", "june",
185
- # "july", "august", "september", "october", "november", "december"], 1)}
186
- # ql = q.lower()
187
- # mon = next((v for k, v in month_map.items() if k in ql), None)
188
- # ym = re.search(r"(19|20)\d{2}", q)
189
- # yr = int(ym.group()) if ym else None
190
- # return mon, yr
191
-
192
- # def extract_topic_match(query, df):
193
- # query_lower = query.lower()
194
- # matched_rows = df[
195
- # df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
196
- # df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
197
- # df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
198
- # df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
199
- # ]
200
- # return matched_rows
201
-
202
-
203
- # st.set_page_config(page_title="IL Trends Q&A", layout="wide")
204
- # st.title("Illinois Legislative Trends Q&A")
205
- # st.markdown("Ask about trends in **topics** like education, higher education, etc!")
206
-
207
- # df = load_data()
208
- # embed_model, summarizer = load_models()
209
-
210
- # query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
211
-
212
- # if query:
213
- # mon, yr = extract_month_year(query)
214
- # df2 = extract_topic_match(query, df)
215
-
216
- # if df2.empty:
217
- # df2 = df
218
- # if yr:
219
- # df2 = df2[df2['status_date'].dt.year == yr]
220
- # if mon:
221
- # df2 = df2[df2['status_date'].dt.month == mon]
222
- # st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
223
- # else:
224
- # st.info(f"Filtering by year: **{yr}**")
225
-
226
- # if df2.empty:
227
- # st.warning("No matching records found.")
228
- # else:
229
- # texts = df2['summary_insight'].tolist()
230
- # embs = compute_embeddings(texts, _model=embed_model)
231
- # res = semantic_search(query, embs, embed_model)
232
-
233
- # if not res:
234
- # st.warning("No relevant insights found.")
235
- # else:
236
- # st.subheader(" Top Matching Insights")
237
- # collected = []
238
-
239
- # for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
240
- # row = df2.iloc[idx]
241
- # date = row['status_date'].date()
242
- # cat_std = row['category_&_subcategory_standardized']
243
- # goal = row['legislative_goal_standardized']
244
- # intent = row['intent_standardized']
245
- # stance = row['stance_standardized']
246
- # trend_summary = row['llama_trend_summary'].strip()
247
-
248
- # st.markdown(f"- ** Date:** {date} | ** Score:** {score:.2f}")
249
- # st.markdown(f" - ** Category:** {cat_std}")
250
- # st.markdown(f" - ** Goal:** {goal}")
251
- # st.markdown(f" - ** Intent:** {intent} | ** Stance:** {stance}")
252
- # st.markdown(f" > ** Trend Summary:** {trend_summary}")
253
-
254
- # collected.append(row['summary_insight'])
255
-
256
- # st.subheader(" RAG-Generated Summary")
257
- # summary = rag_summarize(collected, summarizer)
258
- # st.success(summary)
259
-
260
  import streamlit as st
261
  import pandas as pd
262
  import re
@@ -266,7 +139,7 @@ from sklearn.metrics.pairwise import cosine_similarity
266
  from sklearn.feature_extraction.text import TfidfVectorizer
267
  from datetime import datetime
268
 
269
- # Load data
270
  @st.cache_data
271
  def load_data():
272
  df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
@@ -277,25 +150,22 @@ def load_data():
277
  df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
278
  return df
279
 
280
- # Load models
281
  @st.cache_resource
282
  def load_models():
283
  embed_model = SentenceTransformer('all-MiniLM-L6-v2')
284
  summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
285
  return embed_model, summarizer
286
 
287
- # Compute embeddings
288
  @st.cache_data
289
  def compute_embeddings(texts, _model):
290
  return _model.encode(texts, show_progress_bar=True)
291
 
292
- # Semantic search
293
- def semantic_search(query, embeddings, model, threshold=0.5):
294
  query_embedding = model.encode([query])
295
  sims = cosine_similarity(query_embedding, embeddings)[0]
296
  return [(i, s) for i, s in enumerate(sims) if s > threshold]
297
 
298
- # RAG summarization
299
  def rag_summarize(texts, summarizer, top_k=5):
300
  if not texts:
301
  return "No relevant content to summarize."
@@ -309,61 +179,49 @@ def rag_summarize(texts, summarizer, top_k=5):
309
  out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
310
  return out[0]['summary_text']
311
 
312
- # Extract month and year from query (without dateparser)
313
- def extract_month_year(query):
314
- month_map = {
315
- "january": 1, "february": 2, "march": 3, "april": 4,
316
- "may": 5, "june": 6, "july": 7, "august": 8,
317
- "september": 9, "october": 10, "november": 11, "december": 12
318
- }
 
 
 
 
319
  query_lower = query.lower()
320
- found_month = None
321
- for month_name, month_num in month_map.items():
322
- if month_name in query_lower:
323
- found_month = month_num
324
- break
 
 
325
 
326
- year_match = re.search(r"(19|20)\d{2}", query)
327
- found_year = int(year_match.group()) if year_match else None
328
 
329
- return found_month, found_year
330
-
331
- # Auto-detect category from query
332
- def extract_category(query, categories):
333
- query_lower = query.lower()
334
- for cat in categories:
335
- if pd.isna(cat):
336
- continue
337
- # check if any word in category appears in query
338
- if any(token in query_lower for token in cat.lower().split()):
339
- return cat
340
- return None
341
-
342
- # Streamlit UI
343
  st.set_page_config(page_title="IL Trends Q&A", layout="wide")
344
  st.title("Illinois Legislative Trends Q&A")
 
345
 
346
  df = load_data()
347
  embed_model, summarizer = load_models()
348
 
349
- query = st.text_input("Ask a question (e.g., ‘education in May 2024’):")
350
 
351
  if query:
352
  mon, yr = extract_month_year(query)
353
- cats = df['category_&_subcategory_standardized'].unique()
354
- cat = extract_category(query, cats)
355
 
356
- df2 = df.copy()
357
- if cat:
358
- df2 = df2[df2['category_&_subcategory_standardized'] == cat]
359
- st.info(f"🔎 Filtering by category: **{cat}**")
360
  if yr:
361
  df2 = df2[df2['status_date'].dt.year == yr]
362
  if mon:
363
  df2 = df2[df2['status_date'].dt.month == mon]
364
- st.info(f"🔎 Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
365
  else:
366
- st.info(f"🔎 Filtering by year: **{yr}**")
367
 
368
  if df2.empty:
369
  st.warning("No matching records found.")
@@ -375,7 +233,7 @@ if query:
375
  if not res:
376
  st.warning("No relevant insights found.")
377
  else:
378
- st.subheader("Top Matching Insights")
379
  collected = []
380
 
381
  for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
@@ -387,16 +245,14 @@ if query:
387
  stance = row['stance_standardized']
388
  trend_summary = row['llama_trend_summary'].strip()
389
 
390
- st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
391
- st.markdown(f" - **Category:** {cat_std}")
392
- st.markdown(f" - **Goal:** {goal}")
393
- st.markdown(f" - **Intent:** {intent} | **Stance:** {stance}")
394
- st.markdown(f" > **Trend Summary:** {trend_summary}")
395
 
396
  collected.append(row['summary_insight'])
397
 
398
- st.subheader("RAG-Generated Summary")
399
  summary = rag_summarize(collected, summarizer)
400
  st.success(summary)
401
-
402
-
 
130
  # # summary = rag_summarize(collected, summarizer)
131
  # # st.success(summary)
132
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  import streamlit as st
134
  import pandas as pd
135
  import re
 
139
  from sklearn.feature_extraction.text import TfidfVectorizer
140
  from datetime import datetime
141
 
142
+ # loading data
143
  @st.cache_data
144
  def load_data():
145
  df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
 
150
  df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
151
  return df
152
 
 
153
  @st.cache_resource
154
  def load_models():
155
  embed_model = SentenceTransformer('all-MiniLM-L6-v2')
156
  summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
157
  return embed_model, summarizer
158
 
 
159
  @st.cache_data
160
  def compute_embeddings(texts, _model):
161
  return _model.encode(texts, show_progress_bar=True)
162
 
163
+ def semantic_search(query, embeddings, model, threshold=0.5):
 
164
  query_embedding = model.encode([query])
165
  sims = cosine_similarity(query_embedding, embeddings)[0]
166
  return [(i, s) for i, s in enumerate(sims) if s > threshold]
167
 
168
+
169
  def rag_summarize(texts, summarizer, top_k=5):
170
  if not texts:
171
  return "No relevant content to summarize."
 
179
  out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
180
  return out[0]['summary_text']
181
 
182
+ def extract_month_year(q):
183
+ month_map = {m: i for i, m in enumerate(
184
+ ["january", "february", "march", "april", "may", "june",
185
+ "july", "august", "september", "october", "november", "december"], 1)}
186
+ ql = q.lower()
187
+ mon = next((v for k, v in month_map.items() if k in ql), None)
188
+ ym = re.search(r"(19|20)\d{2}", q)
189
+ yr = int(ym.group()) if ym else None
190
+ return mon, yr
191
+
192
+ def extract_topic_match(query, df):
193
  query_lower = query.lower()
194
+ matched_rows = df[
195
+ df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
196
+ df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
197
+ df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
198
+ df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
199
+ ]
200
+ return matched_rows
201
 
 
 
202
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
203
  st.set_page_config(page_title="IL Trends Q&A", layout="wide")
204
  st.title("Illinois Legislative Trends Q&A")
205
+ st.markdown("Ask about trends in **topics** like education, higher education, etc!")
206
 
207
  df = load_data()
208
  embed_model, summarizer = load_models()
209
 
210
+ query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
211
 
212
  if query:
213
  mon, yr = extract_month_year(query)
214
+ df2 = extract_topic_match(query, df)
 
215
 
216
+ if df2.empty:
217
+ df2 = df
 
 
218
  if yr:
219
  df2 = df2[df2['status_date'].dt.year == yr]
220
  if mon:
221
  df2 = df2[df2['status_date'].dt.month == mon]
222
+ st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
223
  else:
224
+ st.info(f"Filtering by year: **{yr}**")
225
 
226
  if df2.empty:
227
  st.warning("No matching records found.")
 
233
  if not res:
234
  st.warning("No relevant insights found.")
235
  else:
236
+ st.subheader(" Top Matching Insights")
237
  collected = []
238
 
239
  for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
 
245
  stance = row['stance_standardized']
246
  trend_summary = row['llama_trend_summary'].strip()
247
 
248
+ st.markdown(f"- ** Date:** {date} | ** Score:** {score:.2f}")
249
+ st.markdown(f" - ** Category:** {cat_std}")
250
+ st.markdown(f" - ** Goal:** {goal}")
251
+ st.markdown(f" - ** Intent:** {intent} | ** Stance:** {stance}")
252
+ st.markdown(f" > ** Trend Summary:** {trend_summary}")
253
 
254
  collected.append(row['summary_insight'])
255
 
256
+ st.subheader(" RAG-Generated Summary")
257
  summary = rag_summarize(collected, summarizer)
258
  st.success(summary)