tjl8 commited on
Commit
7bbfa1d
·
verified ·
1 Parent(s): 2244d5d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +62 -58
app.py CHANGED
@@ -257,17 +257,14 @@
257
  # summary = rag_summarize(collected, summarizer)
258
  # st.success(summary)
259
 
260
-
261
  import streamlit as st
262
  import pandas as pd
263
  import re
264
- import dateparser # for natural language date parsing
265
  from sentence_transformers import SentenceTransformer
266
  from transformers import pipeline
267
  from sklearn.metrics.pairwise import cosine_similarity
268
  from sklearn.feature_extraction.text import TfidfVectorizer
269
  from datetime import datetime
270
- from io import StringIO
271
 
272
  # Load data
273
  @st.cache_data
@@ -293,7 +290,7 @@ def compute_embeddings(texts, _model):
293
  return _model.encode(texts, show_progress_bar=True)
294
 
295
  # Semantic search
296
- def semantic_search(query, embeddings, model, threshold=0.5):
297
  query_embedding = model.encode([query])
298
  sims = cosine_similarity(query_embedding, embeddings)[0]
299
  return [(i, s) for i, s in enumerate(sims) if s > threshold]
@@ -312,87 +309,94 @@ def rag_summarize(texts, summarizer, top_k=5):
312
  out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
313
  return out[0]['summary_text']
314
 
315
- # Enhanced date parsing with dateparser for flexible queries
316
- def parse_date_from_query(query):
317
- dt = dateparser.parse(query, settings={'PREFER_DATES_FROM': 'past'})
318
- if dt:
319
- return dt.year, dt.month
320
- # fallback: regex extract year and month names
321
- month_map = {m: i for i, m in enumerate(
322
- ["january", "february", "march", "april", "may", "june",
323
- "july", "august", "september", "october", "november", "december"], 1)}
324
- ql = query.lower()
325
- mon = next((v for k, v in month_map.items() if k in ql), None)
326
- ym = re.search(r"(19|20)\d{2}", query)
327
- yr = int(ym.group()) if ym else None
328
- return yr, mon
329
-
330
- # Simple keyword highlighter
331
- def highlight_keywords(text, keywords):
332
- for kw in keywords:
333
- text = re.sub(f"(?i)({re.escape(kw)})", r"**\1**", text)
334
- return text
 
 
 
 
 
 
 
 
 
335
 
336
  # Streamlit UI
337
- st.set_page_config(page_title="IL Trends Q&A Enhanced", layout="wide")
338
- st.title("Illinois Legislative Trends Q&A with Extras")
339
 
340
- # Load data & models
341
  df = load_data()
342
  embed_model, summarizer = load_models()
343
 
344
- query = st.text_input("Ask a question (e.g., ‘education in May 2024’, ‘Opposed bills on healthcare’):")
345
 
346
  if query:
347
- year, month = parse_date_from_query(query)
348
-
349
- # Filter by date if detected
350
- df_filtered = df.copy()
351
- if year:
352
- df_filtered = df_filtered[df_filtered['status_date'].dt.year == year]
353
- if month:
354
- df_filtered = df_filtered[df_filtered['status_date'].dt.month == month]
 
 
 
 
 
 
 
355
 
356
- if df_filtered.empty:
357
- st.warning("No data found for the specified time period.")
358
  else:
359
- # Compute embeddings for filtered data
360
- texts = df_filtered['summary_insight'].tolist()
361
  embs = compute_embeddings(texts, _model=embed_model)
 
362
 
363
- # Perform semantic search with higher threshold
364
- results = semantic_search(query, embs, embed_model, threshold=0.7)
365
-
366
- if not results:
367
  st.warning("No relevant insights found.")
368
  else:
369
  st.subheader("Top Matching Insights")
370
- collected_texts = []
371
- query_keywords = query.lower().split()
372
 
373
- for idx, score in sorted(results, key=lambda x: x[1], reverse=True)[:5]:
374
- row = df_filtered.iloc[idx]
375
  date = row['status_date'].date()
376
- cat = row['category_&_subcategory_standardized']
377
  goal = row['legislative_goal_standardized']
378
  intent = row['intent_standardized']
379
  stance = row['stance_standardized']
380
  trend_summary = row['llama_trend_summary'].strip()
381
- summary_text = row['summary_insight']
382
-
383
- highlighted_summary = highlight_keywords(summary_text, query_keywords)
384
 
385
  st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
386
- st.markdown(f" - **Category:** {cat}")
387
  st.markdown(f" - **Goal:** {goal}")
388
  st.markdown(f" - **Intent:** {intent} | **Stance:** {stance}")
389
  st.markdown(f" > **Trend Summary:** {trend_summary}")
390
- st.markdown(f" > **Summary Insight:** {highlighted_summary}")
391
 
392
- collected_texts.append(summary_text)
393
 
394
- # RAG summary of matched results
395
  st.subheader("RAG-Generated Summary")
396
- rag_summary = rag_summarize(collected_texts, summarizer)
397
- st.success(rag_summary)
 
398
 
 
257
  # summary = rag_summarize(collected, summarizer)
258
  # st.success(summary)
259
 
 
260
  import streamlit as st
261
  import pandas as pd
262
  import re
 
263
  from sentence_transformers import SentenceTransformer
264
  from transformers import pipeline
265
  from sklearn.metrics.pairwise import cosine_similarity
266
  from sklearn.feature_extraction.text import TfidfVectorizer
267
  from datetime import datetime
 
268
 
269
  # Load data
270
  @st.cache_data
 
290
  return _model.encode(texts, show_progress_bar=True)
291
 
292
  # Semantic search
293
+ def semantic_search(query, embeddings, model, threshold=0.7):
294
  query_embedding = model.encode([query])
295
  sims = cosine_similarity(query_embedding, embeddings)[0]
296
  return [(i, s) for i, s in enumerate(sims) if s > threshold]
 
309
  out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
310
  return out[0]['summary_text']
311
 
312
+ # Extract month and year from query (without dateparser)
313
+ def extract_month_year(query):
314
+ month_map = {
315
+ "january": 1, "february": 2, "march": 3, "april": 4,
316
+ "may": 5, "june": 6, "july": 7, "august": 8,
317
+ "september": 9, "october": 10, "november": 11, "december": 12
318
+ }
319
+ query_lower = query.lower()
320
+ found_month = None
321
+ for month_name, month_num in month_map.items():
322
+ if month_name in query_lower:
323
+ found_month = month_num
324
+ break
325
+
326
+ year_match = re.search(r"(19|20)\d{2}", query)
327
+ found_year = int(year_match.group()) if year_match else None
328
+
329
+ return found_month, found_year
330
+
331
+ # Auto-detect category from query
332
+ def extract_category(query, categories):
333
+ query_lower = query.lower()
334
+ for cat in categories:
335
+ if pd.isna(cat):
336
+ continue
337
+ # check if any word in category appears in query
338
+ if any(token in query_lower for token in cat.lower().split()):
339
+ return cat
340
+ return None
341
 
342
  # Streamlit UI
343
+ st.set_page_config(page_title="IL Trends Q&A", layout="wide")
344
+ st.title("Illinois Legislative Trends Q&A")
345
 
 
346
  df = load_data()
347
  embed_model, summarizer = load_models()
348
 
349
+ query = st.text_input("Ask a question (e.g., ‘education in May 2024’):")
350
 
351
  if query:
352
+ mon, yr = extract_month_year(query)
353
+ cats = df['category_&_subcategory_standardized'].unique()
354
+ cat = extract_category(query, cats)
355
+
356
+ df2 = df.copy()
357
+ if cat:
358
+ df2 = df2[df2['category_&_subcategory_standardized'] == cat]
359
+ st.info(f"🔎 Filtering by category: **{cat}**")
360
+ if yr:
361
+ df2 = df2[df2['status_date'].dt.year == yr]
362
+ if mon:
363
+ df2 = df2[df2['status_date'].dt.month == mon]
364
+ st.info(f"🔎 Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
365
+ else:
366
+ st.info(f"🔎 Filtering by year: **{yr}**")
367
 
368
+ if df2.empty:
369
+ st.warning("No matching records found.")
370
  else:
371
+ texts = df2['summary_insight'].tolist()
 
372
  embs = compute_embeddings(texts, _model=embed_model)
373
+ res = semantic_search(query, embs, embed_model)
374
 
375
+ if not res:
 
 
 
376
  st.warning("No relevant insights found.")
377
  else:
378
  st.subheader("Top Matching Insights")
379
+ collected = []
 
380
 
381
+ for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
382
+ row = df2.iloc[idx]
383
  date = row['status_date'].date()
384
+ cat_std = row['category_&_subcategory_standardized']
385
  goal = row['legislative_goal_standardized']
386
  intent = row['intent_standardized']
387
  stance = row['stance_standardized']
388
  trend_summary = row['llama_trend_summary'].strip()
 
 
 
389
 
390
  st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
391
+ st.markdown(f" - **Category:** {cat_std}")
392
  st.markdown(f" - **Goal:** {goal}")
393
  st.markdown(f" - **Intent:** {intent} | **Stance:** {stance}")
394
  st.markdown(f" > **Trend Summary:** {trend_summary}")
 
395
 
396
+ collected.append(row['summary_insight'])
397
 
 
398
  st.subheader("RAG-Generated Summary")
399
+ summary = rag_summarize(collected, summarizer)
400
+ st.success(summary)
401
+
402