Update app.py

app.py

@@ -129,6 +129,134 @@
 # #             st.subheader(" RAG-Generated Summary")
 # #             summary = rag_summarize(collected, summarizer)
 # #             st.success(summary)
+# use the below one
+# import streamlit as st
+# import pandas as pd
+# import re
+# from sentence_transformers import SentenceTransformer
+# from transformers import pipeline
+# from sklearn.metrics.pairwise import cosine_similarity
+# from sklearn.feature_extraction.text import TfidfVectorizer
+# from datetime import datetime
+
+# # loading data 
+# @st.cache_data
+# def load_data():
+#     df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
+#     df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
+#     df = df.dropna(subset=['status_date'])
+#     df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")
+#     df["llama_insight"] = df["llama_insight"].fillna("")
+#     df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
+#     return df
+
+# @st.cache_resource
+# def load_models():
+#     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
+#     summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
+#     return embed_model, summarizer
+
+# @st.cache_data
+# def compute_embeddings(texts, _model):
+#     return _model.encode(texts, show_progress_bar=True)
+
+# def semantic_search(query, embeddings, model, threshold=0.5):  
+#     query_embedding = model.encode([query])
+#     sims = cosine_similarity(query_embedding, embeddings)[0]
+#     return [(i, s) for i, s in enumerate(sims) if s > threshold]
+
+
+# def rag_summarize(texts, summarizer, top_k=5):
+#     if not texts:
+#         return "No relevant content to summarize."
+#     vect = TfidfVectorizer()
+#     m = vect.fit_transform(texts)
+#     mean_vec = m.mean(axis=0).A
+#     scores = cosine_similarity(mean_vec, m).flatten()
+#     top_indices = scores.argsort()[::-1][:top_k]
+#     ctx = "\n".join(texts[i] for i in top_indices)
+#     prompt = "summarize: " + ctx[:1024]
+#     out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
+#     return out[0]['summary_text']
+
+# def extract_month_year(q):
+#     month_map = {m: i for i, m in enumerate(
+#         ["january", "february", "march", "april", "may", "june",
+#          "july", "august", "september", "october", "november", "december"], 1)}
+#     ql = q.lower()
+#     mon = next((v for k, v in month_map.items() if k in ql), None)
+#     ym = re.search(r"(19|20)\d{2}", q)
+#     yr = int(ym.group()) if ym else None
+#     return mon, yr
+
+# def extract_topic_match(query, df):
+#     query_lower = query.lower()
+#     matched_rows = df[
+#         df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
+#         df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
+#         df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
+#         df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
+#     ]
+#     return matched_rows
+
+
+# st.set_page_config(page_title="IL Trends Q&A", layout="wide")
+# st.title("Illinois Legislative Trends Q&A")
+# st.markdown("Ask about trends in **topics** like education, higher education, etc!")
+
+# df = load_data()
+# embed_model, summarizer = load_models()
+
+# query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
+
+# if query:
+#     mon, yr = extract_month_year(query)
+#     df2 = extract_topic_match(query, df)
+
+#     if df2.empty:
+#         df2 = df
+#     if yr:
+#         df2 = df2[df2['status_date'].dt.year == yr]
+#         if mon:
+#             df2 = df2[df2['status_date'].dt.month == mon]
+#             st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
+#         else:
+#             st.info(f"Filtering by year: **{yr}**")
+
+#     if df2.empty:
+#         st.warning("No matching records found.")
+#     else:
+#         texts = df2['summary_insight'].tolist()
+#         embs = compute_embeddings(texts, _model=embed_model)
+#         res = semantic_search(query, embs, embed_model)
+
+#         if not res:
+#             st.warning("No relevant insights found.")
+#         else:
+#             st.subheader(" Top Matching Insights")
+#             collected = []
+
+#             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
+#                 row = df2.iloc[idx]
+#                 date = row['status_date'].date()
+#                 cat_std = row['category_&_subcategory_standardized']
+#                 goal = row['legislative_goal_standardized']
+#                 intent = row['intent_standardized']
+#                 stance = row['stance_standardized']
+#                 trend_summary = row['llama_trend_summary'].strip()
+
+#                 st.markdown(f"- Date: {date} | Score: {score:.2f}")
+#                 st.markdown(f"  -  Category: {cat_std}")
+#                 st.markdown(f"  -  Goal: {goal}")
+#                 st.markdown(f"  -  Intent:{intent} | Stance: {stance}")
+#                 st.markdown(f"  >  Trend Summary: {trend_summary}")
+
+#                 collected.append(row['summary_insight'])
+
+#             st.subheader(" RAG-Generated Summary")
+#             summary = rag_summarize(collected, summarizer)
+#             st.success(summary)
+
 
 import streamlit as st
 import pandas as pd
@@ -139,7 +267,7 @@ from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.feature_extraction.text import TfidfVectorizer
 from datetime import datetime
 
-# loading data 
+# Load the legislative dataset
 @st.cache_data
 def load_data():
     df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
@@ -150,22 +278,25 @@ def load_data():
     df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
     return df
 
+# Load embedding and summarization models
 @st.cache_resource
 def load_models():
     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
     summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
     return embed_model, summarizer
 
+# Compute embeddings for given text corpus
 @st.cache_data
 def compute_embeddings(texts, _model):
     return _model.encode(texts, show_progress_bar=True)
 
-def semantic_search(query, embeddings, model, threshold=0.5):  
+# Perform semantic search to find relevant text chunks
+def semantic_search(query, embeddings, model, threshold=0.5):
     query_embedding = model.encode([query])
     sims = cosine_similarity(query_embedding, embeddings)[0]
     return [(i, s) for i, s in enumerate(sims) if s > threshold]
 
-
+# Use TF-IDF to extract and summarize top-k relevant sections
 def rag_summarize(texts, summarizer, top_k=5):
     if not texts:
         return "No relevant content to summarize."
@@ -179,6 +310,7 @@ def rag_summarize(texts, summarizer, top_k=5):
     out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
     return out[0]['summary_text']
 
+# Extract month and year from query string
 def extract_month_year(q):
     month_map = {m: i for i, m in enumerate(
         ["january", "february", "march", "april", "may", "june",
@@ -189,6 +321,7 @@ def extract_month_year(q):
     yr = int(ym.group()) if ym else None
     return mon, yr
 
+# Extract matching rows based on topic keywords
 def extract_topic_match(query, df):
     query_lower = query.lower()
     matched_rows = df[
@@ -199,14 +332,16 @@ def extract_topic_match(query, df):
     ]
     return matched_rows
 
-
+# Streamlit app UI
 st.set_page_config(page_title="IL Trends Q&A", layout="wide")
 st.title("Illinois Legislative Trends Q&A")
-st.markdown("Ask about trends in **topics** like education, higher education, etc!")
+st.markdown("Ask about trends in **topics** like education, higher education, healthcare, etc.")
 
+# Load data and models
 df = load_data()
 embed_model, summarizer = load_models()
 
+# Query input box
 query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
 
 if query:
@@ -214,7 +349,8 @@ if query:
     df2 = extract_topic_match(query, df)
 
     if df2.empty:
-        df2 = df
+        df2 = df  # fallback to full dataset if nothing matched
+
     if yr:
         df2 = df2[df2['status_date'].dt.year == yr]
         if mon:
@@ -233,7 +369,7 @@ if query:
         if not res:
             st.warning("No relevant insights found.")
         else:
-            st.subheader(" Top Matching Insights")
+            st.subheader("🔎 Top Matching Insights")
             collected = []
 
             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
@@ -244,15 +380,19 @@ if query:
                 intent = row['intent_standardized']
                 stance = row['stance_standardized']
                 trend_summary = row['llama_trend_summary'].strip()
+                bill_id = row['bill_id']
+                url = row['state_link']
 
-                st.markdown(f"- Date: {date} | Score: {score:.2f}")
-                st.markdown(f"  -  Category: {cat_std}")
-                st.markdown(f"  -  Goal: {goal}")
-                st.markdown(f"  -  Intent:{intent} | Stance: {stance}")
-                st.markdown(f"  >  Trend Summary: {trend_summary}")
+                st.markdown(f"---")
+                st.markdown(f"**📅 Date:** {date} | **📄 Bill ID:** `{bill_id}` | **Score:** {score:.2f}")
+                st.markdown(f"**Category:** {cat_std}")
+                st.markdown(f"**Goal:** {goal}")
+                st.markdown(f"**Intent:** {intent} | **Stance:** {stance}")
+                st.markdown(f"**Trend Summary:** {trend_summary}")
+                st.markdown(f"[🔗 View Full Bill]({url})")
 
                 collected.append(row['summary_insight'])
 
-            st.subheader(" RAG-Generated Summary")
+            st.subheader("🧠 RAG-Generated Summary")
             summary = rag_summarize(collected, summarizer)
             st.success(summary)