Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -130,133 +130,6 @@
|
|
| 130 |
# # summary = rag_summarize(collected, summarizer)
|
| 131 |
# # st.success(summary)
|
| 132 |
|
| 133 |
-
# import streamlit as st
|
| 134 |
-
# import pandas as pd
|
| 135 |
-
# import re
|
| 136 |
-
# from sentence_transformers import SentenceTransformer
|
| 137 |
-
# from transformers import pipeline
|
| 138 |
-
# from sklearn.metrics.pairwise import cosine_similarity
|
| 139 |
-
# from sklearn.feature_extraction.text import TfidfVectorizer
|
| 140 |
-
# from datetime import datetime
|
| 141 |
-
|
| 142 |
-
# # loading data
|
| 143 |
-
# @st.cache_data
|
| 144 |
-
# def load_data():
|
| 145 |
-
# df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
|
| 146 |
-
# df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
|
| 147 |
-
# df = df.dropna(subset=['status_date'])
|
| 148 |
-
# df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")
|
| 149 |
-
# df["llama_insight"] = df["llama_insight"].fillna("")
|
| 150 |
-
# df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
|
| 151 |
-
# return df
|
| 152 |
-
|
| 153 |
-
# @st.cache_resource
|
| 154 |
-
# def load_models():
|
| 155 |
-
# embed_model = SentenceTransformer('all-MiniLM-L6-v2')
|
| 156 |
-
# summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
|
| 157 |
-
# return embed_model, summarizer
|
| 158 |
-
|
| 159 |
-
# @st.cache_data
|
| 160 |
-
# def compute_embeddings(texts, _model):
|
| 161 |
-
# return _model.encode(texts, show_progress_bar=True)
|
| 162 |
-
|
| 163 |
-
# def semantic_search(query, embeddings, model, threshold=0.5):
|
| 164 |
-
# query_embedding = model.encode([query])
|
| 165 |
-
# sims = cosine_similarity(query_embedding, embeddings)[0]
|
| 166 |
-
# return [(i, s) for i, s in enumerate(sims) if s > threshold]
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
# def rag_summarize(texts, summarizer, top_k=5):
|
| 170 |
-
# if not texts:
|
| 171 |
-
# return "No relevant content to summarize."
|
| 172 |
-
# vect = TfidfVectorizer()
|
| 173 |
-
# m = vect.fit_transform(texts)
|
| 174 |
-
# mean_vec = m.mean(axis=0).A
|
| 175 |
-
# scores = cosine_similarity(mean_vec, m).flatten()
|
| 176 |
-
# top_indices = scores.argsort()[::-1][:top_k]
|
| 177 |
-
# ctx = "\n".join(texts[i] for i in top_indices)
|
| 178 |
-
# prompt = "summarize: " + ctx[:1024]
|
| 179 |
-
# out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
|
| 180 |
-
# return out[0]['summary_text']
|
| 181 |
-
|
| 182 |
-
# def extract_month_year(q):
|
| 183 |
-
# month_map = {m: i for i, m in enumerate(
|
| 184 |
-
# ["january", "february", "march", "april", "may", "june",
|
| 185 |
-
# "july", "august", "september", "october", "november", "december"], 1)}
|
| 186 |
-
# ql = q.lower()
|
| 187 |
-
# mon = next((v for k, v in month_map.items() if k in ql), None)
|
| 188 |
-
# ym = re.search(r"(19|20)\d{2}", q)
|
| 189 |
-
# yr = int(ym.group()) if ym else None
|
| 190 |
-
# return mon, yr
|
| 191 |
-
|
| 192 |
-
# def extract_topic_match(query, df):
|
| 193 |
-
# query_lower = query.lower()
|
| 194 |
-
# matched_rows = df[
|
| 195 |
-
# df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
|
| 196 |
-
# df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
|
| 197 |
-
# df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
|
| 198 |
-
# df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
|
| 199 |
-
# ]
|
| 200 |
-
# return matched_rows
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
# st.set_page_config(page_title="IL Trends Q&A", layout="wide")
|
| 204 |
-
# st.title("Illinois Legislative Trends Q&A")
|
| 205 |
-
# st.markdown("Ask about trends in **topics** like education, higher education, etc!")
|
| 206 |
-
|
| 207 |
-
# df = load_data()
|
| 208 |
-
# embed_model, summarizer = load_models()
|
| 209 |
-
|
| 210 |
-
# query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
|
| 211 |
-
|
| 212 |
-
# if query:
|
| 213 |
-
# mon, yr = extract_month_year(query)
|
| 214 |
-
# df2 = extract_topic_match(query, df)
|
| 215 |
-
|
| 216 |
-
# if df2.empty:
|
| 217 |
-
# df2 = df
|
| 218 |
-
# if yr:
|
| 219 |
-
# df2 = df2[df2['status_date'].dt.year == yr]
|
| 220 |
-
# if mon:
|
| 221 |
-
# df2 = df2[df2['status_date'].dt.month == mon]
|
| 222 |
-
# st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
|
| 223 |
-
# else:
|
| 224 |
-
# st.info(f"Filtering by year: **{yr}**")
|
| 225 |
-
|
| 226 |
-
# if df2.empty:
|
| 227 |
-
# st.warning("No matching records found.")
|
| 228 |
-
# else:
|
| 229 |
-
# texts = df2['summary_insight'].tolist()
|
| 230 |
-
# embs = compute_embeddings(texts, _model=embed_model)
|
| 231 |
-
# res = semantic_search(query, embs, embed_model)
|
| 232 |
-
|
| 233 |
-
# if not res:
|
| 234 |
-
# st.warning("No relevant insights found.")
|
| 235 |
-
# else:
|
| 236 |
-
# st.subheader(" Top Matching Insights")
|
| 237 |
-
# collected = []
|
| 238 |
-
|
| 239 |
-
# for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
|
| 240 |
-
# row = df2.iloc[idx]
|
| 241 |
-
# date = row['status_date'].date()
|
| 242 |
-
# cat_std = row['category_&_subcategory_standardized']
|
| 243 |
-
# goal = row['legislative_goal_standardized']
|
| 244 |
-
# intent = row['intent_standardized']
|
| 245 |
-
# stance = row['stance_standardized']
|
| 246 |
-
# trend_summary = row['llama_trend_summary'].strip()
|
| 247 |
-
|
| 248 |
-
# st.markdown(f"- ** Date:** {date} | ** Score:** {score:.2f}")
|
| 249 |
-
# st.markdown(f" - ** Category:** {cat_std}")
|
| 250 |
-
# st.markdown(f" - ** Goal:** {goal}")
|
| 251 |
-
# st.markdown(f" - ** Intent:** {intent} | ** Stance:** {stance}")
|
| 252 |
-
# st.markdown(f" > ** Trend Summary:** {trend_summary}")
|
| 253 |
-
|
| 254 |
-
# collected.append(row['summary_insight'])
|
| 255 |
-
|
| 256 |
-
# st.subheader(" RAG-Generated Summary")
|
| 257 |
-
# summary = rag_summarize(collected, summarizer)
|
| 258 |
-
# st.success(summary)
|
| 259 |
-
|
| 260 |
import streamlit as st
|
| 261 |
import pandas as pd
|
| 262 |
import re
|
|
@@ -266,7 +139,7 @@ from sklearn.metrics.pairwise import cosine_similarity
|
|
| 266 |
from sklearn.feature_extraction.text import TfidfVectorizer
|
| 267 |
from datetime import datetime
|
| 268 |
|
| 269 |
-
#
|
| 270 |
@st.cache_data
|
| 271 |
def load_data():
|
| 272 |
df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
|
|
@@ -277,25 +150,22 @@ def load_data():
|
|
| 277 |
df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
|
| 278 |
return df
|
| 279 |
|
| 280 |
-
# Load models
|
| 281 |
@st.cache_resource
|
| 282 |
def load_models():
|
| 283 |
embed_model = SentenceTransformer('all-MiniLM-L6-v2')
|
| 284 |
summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
|
| 285 |
return embed_model, summarizer
|
| 286 |
|
| 287 |
-
# Compute embeddings
|
| 288 |
@st.cache_data
|
| 289 |
def compute_embeddings(texts, _model):
|
| 290 |
return _model.encode(texts, show_progress_bar=True)
|
| 291 |
|
| 292 |
-
|
| 293 |
-
def semantic_search(query, embeddings, model, threshold=0.5):
|
| 294 |
query_embedding = model.encode([query])
|
| 295 |
sims = cosine_similarity(query_embedding, embeddings)[0]
|
| 296 |
return [(i, s) for i, s in enumerate(sims) if s > threshold]
|
| 297 |
|
| 298 |
-
|
| 299 |
def rag_summarize(texts, summarizer, top_k=5):
|
| 300 |
if not texts:
|
| 301 |
return "No relevant content to summarize."
|
|
@@ -309,61 +179,49 @@ def rag_summarize(texts, summarizer, top_k=5):
|
|
| 309 |
out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
|
| 310 |
return out[0]['summary_text']
|
| 311 |
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
query_lower = query.lower()
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
|
|
|
|
|
|
| 325 |
|
| 326 |
-
year_match = re.search(r"(19|20)\d{2}", query)
|
| 327 |
-
found_year = int(year_match.group()) if year_match else None
|
| 328 |
|
| 329 |
-
return found_month, found_year
|
| 330 |
-
|
| 331 |
-
# Auto-detect category from query
|
| 332 |
-
def extract_category(query, categories):
|
| 333 |
-
query_lower = query.lower()
|
| 334 |
-
for cat in categories:
|
| 335 |
-
if pd.isna(cat):
|
| 336 |
-
continue
|
| 337 |
-
# check if any word in category appears in query
|
| 338 |
-
if any(token in query_lower for token in cat.lower().split()):
|
| 339 |
-
return cat
|
| 340 |
-
return None
|
| 341 |
-
|
| 342 |
-
# Streamlit UI
|
| 343 |
st.set_page_config(page_title="IL Trends Q&A", layout="wide")
|
| 344 |
st.title("Illinois Legislative Trends Q&A")
|
|
|
|
| 345 |
|
| 346 |
df = load_data()
|
| 347 |
embed_model, summarizer = load_models()
|
| 348 |
|
| 349 |
-
query = st.text_input("Ask a question (e.g., ‘
|
| 350 |
|
| 351 |
if query:
|
| 352 |
mon, yr = extract_month_year(query)
|
| 353 |
-
|
| 354 |
-
cat = extract_category(query, cats)
|
| 355 |
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
df2 = df2[df2['category_&_subcategory_standardized'] == cat]
|
| 359 |
-
st.info(f"🔎 Filtering by category: **{cat}**")
|
| 360 |
if yr:
|
| 361 |
df2 = df2[df2['status_date'].dt.year == yr]
|
| 362 |
if mon:
|
| 363 |
df2 = df2[df2['status_date'].dt.month == mon]
|
| 364 |
-
st.info(f"
|
| 365 |
else:
|
| 366 |
-
st.info(f"
|
| 367 |
|
| 368 |
if df2.empty:
|
| 369 |
st.warning("No matching records found.")
|
|
@@ -375,7 +233,7 @@ if query:
|
|
| 375 |
if not res:
|
| 376 |
st.warning("No relevant insights found.")
|
| 377 |
else:
|
| 378 |
-
st.subheader("Top Matching Insights")
|
| 379 |
collected = []
|
| 380 |
|
| 381 |
for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
|
|
@@ -387,16 +245,14 @@ if query:
|
|
| 387 |
stance = row['stance_standardized']
|
| 388 |
trend_summary = row['llama_trend_summary'].strip()
|
| 389 |
|
| 390 |
-
st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
|
| 391 |
-
st.markdown(f" - **Category:** {cat_std}")
|
| 392 |
-
st.markdown(f" - **Goal:** {goal}")
|
| 393 |
-
st.markdown(f" - **Intent:** {intent} | **Stance:** {stance}")
|
| 394 |
-
st.markdown(f" > **Trend Summary:** {trend_summary}")
|
| 395 |
|
| 396 |
collected.append(row['summary_insight'])
|
| 397 |
|
| 398 |
-
st.subheader("RAG-Generated Summary")
|
| 399 |
summary = rag_summarize(collected, summarizer)
|
| 400 |
st.success(summary)
|
| 401 |
-
|
| 402 |
-
|
|
|
|
| 130 |
# # summary = rag_summarize(collected, summarizer)
|
| 131 |
# # st.success(summary)
|
| 132 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
import streamlit as st
|
| 134 |
import pandas as pd
|
| 135 |
import re
|
|
|
|
| 139 |
from sklearn.feature_extraction.text import TfidfVectorizer
|
| 140 |
from datetime import datetime
|
| 141 |
|
| 142 |
+
# loading data
|
| 143 |
@st.cache_data
|
| 144 |
def load_data():
|
| 145 |
df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
|
|
|
|
| 150 |
df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
|
| 151 |
return df
|
| 152 |
|
|
|
|
| 153 |
@st.cache_resource
|
| 154 |
def load_models():
|
| 155 |
embed_model = SentenceTransformer('all-MiniLM-L6-v2')
|
| 156 |
summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
|
| 157 |
return embed_model, summarizer
|
| 158 |
|
|
|
|
| 159 |
@st.cache_data
|
| 160 |
def compute_embeddings(texts, _model):
|
| 161 |
return _model.encode(texts, show_progress_bar=True)
|
| 162 |
|
| 163 |
+
def semantic_search(query, embeddings, model, threshold=0.5):
|
|
|
|
| 164 |
query_embedding = model.encode([query])
|
| 165 |
sims = cosine_similarity(query_embedding, embeddings)[0]
|
| 166 |
return [(i, s) for i, s in enumerate(sims) if s > threshold]
|
| 167 |
|
| 168 |
+
|
| 169 |
def rag_summarize(texts, summarizer, top_k=5):
|
| 170 |
if not texts:
|
| 171 |
return "No relevant content to summarize."
|
|
|
|
| 179 |
out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
|
| 180 |
return out[0]['summary_text']
|
| 181 |
|
| 182 |
+
def extract_month_year(q):
|
| 183 |
+
month_map = {m: i for i, m in enumerate(
|
| 184 |
+
["january", "february", "march", "april", "may", "june",
|
| 185 |
+
"july", "august", "september", "october", "november", "december"], 1)}
|
| 186 |
+
ql = q.lower()
|
| 187 |
+
mon = next((v for k, v in month_map.items() if k in ql), None)
|
| 188 |
+
ym = re.search(r"(19|20)\d{2}", q)
|
| 189 |
+
yr = int(ym.group()) if ym else None
|
| 190 |
+
return mon, yr
|
| 191 |
+
|
| 192 |
+
def extract_topic_match(query, df):
|
| 193 |
query_lower = query.lower()
|
| 194 |
+
matched_rows = df[
|
| 195 |
+
df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
|
| 196 |
+
df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
|
| 197 |
+
df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
|
| 198 |
+
df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
|
| 199 |
+
]
|
| 200 |
+
return matched_rows
|
| 201 |
|
|
|
|
|
|
|
| 202 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
st.set_page_config(page_title="IL Trends Q&A", layout="wide")
|
| 204 |
st.title("Illinois Legislative Trends Q&A")
|
| 205 |
+
st.markdown("Ask about trends in **topics** like education, higher education, etc!")
|
| 206 |
|
| 207 |
df = load_data()
|
| 208 |
embed_model, summarizer = load_models()
|
| 209 |
|
| 210 |
+
query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
|
| 211 |
|
| 212 |
if query:
|
| 213 |
mon, yr = extract_month_year(query)
|
| 214 |
+
df2 = extract_topic_match(query, df)
|
|
|
|
| 215 |
|
| 216 |
+
if df2.empty:
|
| 217 |
+
df2 = df
|
|
|
|
|
|
|
| 218 |
if yr:
|
| 219 |
df2 = df2[df2['status_date'].dt.year == yr]
|
| 220 |
if mon:
|
| 221 |
df2 = df2[df2['status_date'].dt.month == mon]
|
| 222 |
+
st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
|
| 223 |
else:
|
| 224 |
+
st.info(f"Filtering by year: **{yr}**")
|
| 225 |
|
| 226 |
if df2.empty:
|
| 227 |
st.warning("No matching records found.")
|
|
|
|
| 233 |
if not res:
|
| 234 |
st.warning("No relevant insights found.")
|
| 235 |
else:
|
| 236 |
+
st.subheader(" Top Matching Insights")
|
| 237 |
collected = []
|
| 238 |
|
| 239 |
for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
|
|
|
|
| 245 |
stance = row['stance_standardized']
|
| 246 |
trend_summary = row['llama_trend_summary'].strip()
|
| 247 |
|
| 248 |
+
st.markdown(f"- ** Date:** {date} | ** Score:** {score:.2f}")
|
| 249 |
+
st.markdown(f" - ** Category:** {cat_std}")
|
| 250 |
+
st.markdown(f" - ** Goal:** {goal}")
|
| 251 |
+
st.markdown(f" - ** Intent:** {intent} | ** Stance:** {stance}")
|
| 252 |
+
st.markdown(f" > ** Trend Summary:** {trend_summary}")
|
| 253 |
|
| 254 |
collected.append(row['summary_insight'])
|
| 255 |
|
| 256 |
+
st.subheader(" RAG-Generated Summary")
|
| 257 |
summary = rag_summarize(collected, summarizer)
|
| 258 |
st.success(summary)
|
|
|
|
|
|