File size: 25,582 Bytes
7bf6685 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 | import streamlit as st
import pandas as pd
import sqlalchemy as sql
from langchain_groq import ChatGroq
from langchain.prompts import PromptTemplate
import streamlit.components.v1 as components
from langchain_core.output_parsers import StrOutputParser
import base64
import os
from agents.sql_database_agent import SQLDatabaseAgent
from agents.eda_database_agent import run_visualization_agent
DB_PATH = "uploaded_csv_db.db"
TABLE_NAME = "uploaded_data"
# Page config
st.set_page_config(page_icon="π", layout="wide", page_title="All-in-One Data Chatbot")
st.title("π€ Ask Anything About Your Data")
# Sidebar
with st.sidebar:
st.header("βοΈ Settings")
uploaded_file = st.file_uploader("π Upload a CSV", type=["csv"])
st.markdown("---")
st.header("π Groq API Key")
groq_api_key = st.text_input("Enter your Groq API Key", type="password", key="groq_api_key")
if groq_api_key:
st.session_state["GROQ_API_KEY"] = groq_api_key
else:
groq_api_key = st.session_state.get("GROQ_API_KEY", None)
# --- Ultimate Data Analytics Agent UI ---
# Add sidebar agent/task selector
with st.sidebar:
st.header("π§° Agent/Task Selector")
agent_task = st.radio(
"Select task:",
[
"Data Overview",
"EDA",
"Data Cleaning",
"Visualization",
"Query/Ask",
"AutoML",
"Advanced",
"Report",
"Data Quality",
"Smart Cleaning",
"Auto Insights",
"Pipeline Builder",
"Collaboration",
"Alerts & Monitoring"
],
index=0
)
# Define the prompt template
decision_template = """
You are a smart assistant designed to choose the correct tool for analyzing a dataset.
Your job is to decide which **agent** to activate based on the user's input. Analyze the question and return ONLY one of the following options:
- Agent1 (π§ Data Exploration / Statistics):
- When the user asks for high-level summaries, insights, dataset descriptions, distributions, or overall explanations.
- Examples: "Describe this dataset", "What is the structure?", "Give me a summary", "What is interesting in the data?"
- Agent2 (π Data Visualization):
- When the user asks to visualize or plot the data (heatmap, histogram, pie chart, trends, etc).
- Examples: "Show correlation heatmap", "Plot price trends", "Visualize monthly sales"
- Agent3 (ποΈ SQL Query):
- When the user is asking for precise calculations, aggregations, filtering, or row-level data that can be fetched using SQL queries.
- Look for keywords like: "average", "total", "count", "maximum", "minimum", "select", "filter", "how many", "retrieve", or if dates/time ranges are involved.
- Examples: "What was the average sales in March?", "Get rows where quantity > 100", "Show all orders from 2022"
π Return only **Agent1**, **Agent2**, or **Agent3**. No explanations.
User Input: {user_input}
"""
decision_maker = PromptTemplate(input_variables=["user_input"], template=decision_template)
# Initialize LLM
llm = ChatGroq(
model="llama3-70b-8192",
api_key=groq_api_key or os.getenv("TOKEN"),
)
def upload_csv_to_sqlite(file, db_path: str, table_name: str, conn) -> pd.DataFrame:
df.columns = [col.strip().replace(" ", "_") for col in df.columns]
try:
df.to_sql(table_name, conn, if_exists="replace", index=False)
except Exception as e:
raise RuntimeError(f"Failed to upload data to SQLite: {e}")
return df
def create_download_link(path, filename="sweetviz_report.html"):
with open(path, "rb") as f:
data = f.read()
b64 = base64.b64encode(data).decode()
href = f'<a href="data:text/html;base64,{b64}" download="{filename}">π₯ Download Sweetviz Report</a>'
return href
# If file uploaded
if uploaded_file:
# Load into SQLite
df = pd.read_csv(uploaded_file)
sql_engine = sql.create_engine("sqlite:///" + DB_PATH)
conn = sql_engine.connect()
df = upload_csv_to_sqlite(uploaded_file, DB_PATH, TABLE_NAME, conn)
st.success("β
CSV uploaded and loaded into SQLite!")
st.dataframe(df.head(), use_container_width=True)
st.session_state["data_raw"] = df.to_dict(orient="records")
# Initialize agents
sql_agent = SQLDatabaseAgent(llm=llm, conn=conn)
# --- Data Quality Tab ---
if agent_task == "Data Quality":
st.subheader("Automated Data Quality Assessment")
if uploaded_file:
quality_issues = []
if df.isnull().sum().sum() > 0:
quality_issues.append("Missing values detected.")
print("[Data Quality] Missing values detected.")
if df.duplicated().sum() > 0:
quality_issues.append("Duplicate rows detected.")
print("[Data Quality] Duplicate rows detected.")
if not all([pd.api.types.is_numeric_dtype(df[col]) or pd.api.types.is_string_dtype(df[col]) for col in df.columns]):
quality_issues.append("Inconsistent data types detected.")
print("[Data Quality] Inconsistent data types detected.")
health_score = 100-len(quality_issues)*30
st.write("**Data Health Score:**", health_score)
print(f"[Data Quality] Data Health Score: {health_score}")
st.write("**Issues:**")
for issue in quality_issues:
st.warning(issue)
if not quality_issues:
st.success("No major data quality issues detected!")
print("[Data Quality] No major data quality issues detected!")
else:
st.info("Upload a CSV file to assess data quality.")
print("[Data Quality] No file uploaded.")
# --- Smart Cleaning Tab ---
elif agent_task == "Smart Cleaning":
st.subheader("Smart Data Cleaning")
if uploaded_file:
st.write("Preview and apply recommended cleaning steps:")
if st.button("Drop duplicates"):
df.drop_duplicates(inplace=True)
st.success("Duplicates dropped.")
print("[Smart Cleaning] Duplicates dropped.")
if st.button("Fill missing values (0)"):
df.fillna(0, inplace=True)
st.success("Missing values filled with 0.")
print("[Smart Cleaning] Missing values filled with 0.")
st.dataframe(df.head(), use_container_width=True)
else:
st.info("Upload a CSV file to use smart cleaning.")
print("[Smart Cleaning] No file uploaded.")
# --- Auto Insights Tab ---
elif agent_task == "Auto Insights":
st.subheader("Automated Insights & Recommendations")
if uploaded_file:
st.write("Key trends and recommendations:")
st.write("- Top correlated features:", df.corr(numeric_only=True).abs().sum().sort_values(ascending=False).head(3).to_dict())
st.write("- Row count:", len(df))
st.write("- Column count:", len(df.columns))
st.write("- Sample mean values:", df.mean(numeric_only=True).to_dict())
# Add more insights as needed
else:
st.info("Upload a CSV file to get insights.")
# --- Pipeline Builder Tab ---
elif agent_task == "Pipeline Builder":
st.subheader("Data Pipeline Builder (Prototype)")
st.write("Chain together cleaning, transformation, and modeling steps (coming soon!)")
# --- Collaboration Tab ---
elif agent_task == "Collaboration":
st.subheader("Collaboration Tools")
st.write("Share your analysis session or dashboard with others (coming soon!)")
# --- Alerts & Monitoring Tab ---
elif agent_task == "Alerts & Monitoring":
st.subheader("Alerts & Monitoring")
st.write("Set up alerts for data changes, anomalies, or model drift (coming soon!)")
# Main area: show content based on selected task
else:
tabs = st.tabs([
"Data Overview",
"EDA",
"Data Cleaning",
"Visualization",
"Query/Ask",
"AutoML",
"Advanced",
"Report"
])
with tabs[0]:
st.subheader("Data Overview")
st.dataframe(df.head(), use_container_width=True)
st.write(df.info())
st.write(df.describe(include='all'))
with tabs[1]:
st.subheader("EDA")
from ydata_profiling import ProfileReport
import tempfile
import os
st.write("Generate and explore an automated EDA report for your data.")
if st.button("Run EDA Report"):
with st.spinner("Generating EDA report..."):
profile = ProfileReport(df, title="Pandas Profiling Report", explorative=True)
with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as tmpfile:
profile.to_file(tmpfile.name)
report_path = tmpfile.name
st.success("EDA report generated!")
with open(report_path, "r", encoding="utf-8") as f:
html_content = f.read()
components.html(html_content, height=800, scrolling=True)
# Download link
b64 = base64.b64encode(open(report_path, "rb").read()).decode()
href = f'<a href="data:text/html;base64,{b64}" download="EDA_Report.html">π₯ Download EDA Report</a>'
st.markdown(href, unsafe_allow_html=True)
with tabs[2]:
st.subheader("Data Cleaning")
st.write("Interactively clean and transform your data.")
# Show missing values
st.markdown("#### Missing Values")
missing = df.isnull().sum()
st.dataframe(missing[missing > 0], use_container_width=True)
# Drop missing values
if st.button("Drop rows with missing values"):
df.dropna(inplace=True)
st.success("Rows with missing values dropped.")
# Fill missing values
fill_col = st.selectbox("Select column to fill missing values", [col for col in df.columns if df[col].isnull().sum() > 0], key="fill_col")
fill_value = st.text_input("Fill missing with (value or 'mean', 'median', 'mode')", key="fill_val")
if st.button("Fill missing values") and fill_col and fill_value:
if fill_value.lower() == "mean":
df[fill_col].fillna(df[fill_col].mean(), inplace=True)
elif fill_value.lower() == "median":
df[fill_col].fillna(df[fill_col].median(), inplace=True)
elif fill_value.lower() == "mode":
df[fill_col].fillna(df[fill_col].mode()[0], inplace=True)
else:
df[fill_col].fillna(fill_value, inplace=True)
st.success(f"Filled missing values in {fill_col}.")
# Remove duplicates
if st.button("Remove duplicate rows"):
df.drop_duplicates(inplace=True)
st.success("Duplicate rows removed.")
# Encode categoricals
cat_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
if cat_cols:
enc_col = st.selectbox("Select categorical column to encode", cat_cols, key="enc_col")
if st.button("One-hot encode column"):
df = pd.get_dummies(df, columns=[enc_col])
st.success(f"One-hot encoded {enc_col}.")
# Normalize columns
num_cols = df.select_dtypes(include=['number']).columns.tolist()
norm_col = st.selectbox("Select numeric column to normalize", num_cols, key="norm_col")
if st.button("Normalize column"):
df[norm_col] = (df[norm_col] - df[norm_col].mean()) / df[norm_col].std()
st.success(f"Normalized {norm_col}.")
st.dataframe(df.head(), use_container_width=True)
with tabs[3]:
st.subheader("Visualization")
st.write("Create custom and automated visualizations.")
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
plot_types = ["Histogram", "Scatter", "Bar", "Line", "Box", "Pie"]
plot_type = st.selectbox("Select plot type", plot_types, key="plot_type")
x_col = st.selectbox("X axis", df.columns, key="x_col")
y_col = st.selectbox("Y axis (optional)", [None] + list(df.columns), key="y_col")
group_col = st.selectbox("Group/Color by (optional)", [None] + list(df.columns), key="group_col")
if st.button("Generate Plot"):
fig = None
if plot_type == "Histogram":
fig = px.histogram(df, x=x_col, color=group_col if group_col else None)
elif plot_type == "Scatter" and y_col:
fig = px.scatter(df, x=x_col, y=y_col, color=group_col if group_col else None)
elif plot_type == "Bar" and y_col:
fig = px.bar(df, x=x_col, y=y_col, color=group_col if group_col else None)
elif plot_type == "Line" and y_col:
fig = px.line(df, x=x_col, y=y_col, color=group_col if group_col else None)
elif plot_type == "Box" and y_col:
fig = px.box(df, x=x_col, y=y_col, color=group_col if group_col else None)
elif plot_type == "Pie":
fig = px.pie(df, names=x_col, values=y_col if y_col else None)
if fig:
st.plotly_chart(fig, use_container_width=True)
st.success("Plot generated!")
else:
st.warning("Please select valid columns for the selected plot type.")
with tabs[4]:
st.subheader("Query/Ask")
st.write("Chat with your data using natural language or SQL.")
from streamlit_chat import message
if "chat_history" not in st.session_state:
st.session_state["chat_history"] = []
user_query = st.text_input("Enter your question or SQL query:")
if st.button("Send") and user_query:
st.session_state["chat_history"].append({"role": "user", "content": user_query})
try:
if user_query.strip().lower().startswith("select"):
# SQL mode
result = pd.read_sql_query(user_query, conn)
st.session_state["chat_history"].append({"role": "assistant", "content": result.to_markdown()})
else:
# Natural language mode (LLM)
decision_chain = decision_maker | llm | StrOutputParser()
decision = decision_chain.invoke({"user_input": user_query})
if decision.strip() == "Agent3":
sql_agent.smart_filter_schema(user_query)
sql_query = sql_agent.generate_sql_query(user_query)
result = sql_agent.run_sql_query(sql_query)
sql_response = f"**π§ Generated SQL Query:**\n```sql\n{sql_query}\n```"
if isinstance(result, pd.DataFrame):
sql_response += f"\n**π Query Result:**\n{result.to_markdown()}"
else:
sql_response += f"\n{result}"
st.session_state["chat_history"].append({"role": "assistant", "content": sql_response})
else:
st.session_state["chat_history"].append({"role": "assistant", "content": "LLM: This would route to EDA or Visualization agent. (Stub)"})
except Exception as e:
st.session_state["chat_history"].append({"role": "assistant", "content": f"Error: {e}"})
# Display chat
for i, msg in enumerate(st.session_state["chat_history"]):
is_user = msg["role"] == "user"
message(msg["content"], is_user=is_user, key=f"chat_{i}")
with tabs[5]:
st.subheader("AutoML")
st.write("Train an AutoML model on your data.")
from pycaret.classification import setup as cls_setup, compare_models as cls_compare, predict_model as cls_predict
from pycaret.regression import setup as reg_setup, compare_models as reg_compare, predict_model as reg_predict
target_col = st.selectbox("Select target column", df.columns, key="target_col")
# Feature selection: all columns except target by default
default_features = [col for col in df.columns if col != target_col]
selected_features = st.multiselect("Select features to include", options=[col for col in df.columns if col != target_col], default=default_features, key="feature_cols")
use_all_features = st.checkbox("Use all features for ML", value=True)
task_type = st.selectbox("Task type", ["Classification", "Regression"], key="task_type")
if use_all_features:
selected_features = [col for col in df.columns if col != target_col]
if st.button("Run AutoML"):
# Show progress and logs
progress_bar = st.progress(0)
log_area = st.empty()
with st.spinner("Running AutoML..."):
try:
df_selected = df[selected_features + [target_col]]
log_msgs = []
def log(msg, pct=None):
log_msgs.append(msg)
log_area.code("\n".join(log_msgs))
if pct is not None:
progress_bar.progress(pct)
log("Setting up experiment...", 10)
if task_type == "Classification":
s = cls_setup(df_selected, target=target_col, session_id=123, html=False, log_experiment=False)
log("Comparing models...", 40)
best_model = cls_compare()
if isinstance(best_model, list):
best_model = best_model[0]
log("Predicting on data...", 70)
preds = cls_predict(best_model, data=df_selected)
log("Calculating metrics...", 90)
st.success("Best classification model trained!")
st.write(best_model)
st.dataframe(preds, use_container_width=True)
from pycaret.classification import pull
results = pull()
st.subheader("Model Comparison & Metrics")
st.dataframe(results, use_container_width=True)
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report
y_true = df_selected[target_col]
y_pred = preds['Label'] if 'Label' in preds.columns else preds.iloc[:, -1]
st.write(f"Accuracy: {accuracy_score(y_true, y_pred):.4f}")
st.write(f"Precision: {precision_score(y_true, y_pred, average='weighted', zero_division=0):.4f}")
st.write(f"Recall: {recall_score(y_true, y_pred, average='weighted', zero_division=0):.4f}")
st.write(f"F1 Score: {f1_score(y_true, y_pred, average='weighted', zero_division=0):.4f}")
st.text("\nClassification Report:\n" + classification_report(y_true, y_pred, zero_division=0))
log("Done!", 100)
else:
s = reg_setup(df_selected, target=target_col, session_id=123, html=False, log_experiment=False)
log("Comparing models...", 40)
best_model = reg_compare()
if isinstance(best_model, list):
best_model = best_model[0]
log("Predicting on data...", 70)
preds = reg_predict(best_model, data=df_selected)
log("Calculating metrics...", 90)
st.success("Best regression model trained!")
st.write(best_model)
st.dataframe(preds, use_container_width=True)
from pycaret.regression import pull
results = pull()
st.subheader("Model Comparison & Metrics")
st.dataframe(results, use_container_width=True)
from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error
y_true = df_selected[target_col]
y_pred = preds['Label'] if 'Label' in preds.columns else preds.iloc[:, -1]
st.write(f"R2 Score: {r2_score(y_true, y_pred):.4f}")
st.write(f"MSE: {mean_squared_error(y_true, y_pred):.4f}")
st.write(f"MAE: {mean_absolute_error(y_true, y_pred):.4f}")
log("Done!", 100)
except Exception as e:
st.error(f"AutoML Error: {e}")
log(f"Error: {e}", 100)
with tabs[6]:
st.subheader("Advanced")
st.write("Time series forecasting, clustering, and anomaly detection coming soon!")
# TODO: integrate advanced analytics agents
with tabs[7]:
st.subheader("Report")
st.write("Generate and download a summary report of your analysis. (Coming soon!)")
# TODO: integrate report agent
# User input
st.markdown("### π¬ Ask a question about your data:")
user_question = st.chat_input("Ask anything...")
if user_question:
with st.chat_message("user"):
st.markdown(user_question)
decision_chain = decision_maker | llm | StrOutputParser()
decision = decision_chain.invoke({
"user_input": user_question
})
print("decision = ", decision)
with st.chat_message("assistant"):
with st.spinner("π€ Let me figure that out..."):
try:
# Use prompt to guide routing (lightweight classifier)
if decision.strip() == "Agent1" or decision.strip() == "Agent2":
print("right direction")
# Go to visualization
content, artifact = run_visualization_agent(
conn=conn,
question=user_question,
llm=llm,
data_raw=st.session_state["data_raw"],
decision = decision
)
st.markdown(content)
#print("artifact = ", artifact.type)
if artifact:
if isinstance(artifact, list):
for fig in artifact:
st.plotly_chart(fig, use_container_width=True)
elif isinstance(artifact, dict):
# After you receive content and artifact from the agent
report_path = artifact["report_path"]
# Preview Sweetviz report
with open(report_path, "r", encoding="utf-8") as f:
html_content = f.read()
components.html(html_content, height=800, scrolling=True)
# Add download link
download_link = create_download_link(report_path)
st.markdown(download_link, unsafe_allow_html=True)
else:
st.error("β οΈ Invalid artifact format.")
elif decision.strip() == "Agent3":
print("sql direction")
# Go to SQL
sql_agent.smart_filter_schema(user_question)
sql_query = sql_agent.generate_sql_query(user_question)
result = sql_agent.run_sql_query(sql_query)
st.markdown("**π§ Generated SQL Query:**")
st.code(sql_query, language="sql")
if isinstance(result, pd.DataFrame):
st.markdown("**π Query Result:**")
st.dataframe(result, use_container_width=True)
else:
st.warning(result)
except Exception as e:
st.error(f"β Error: {e}")
|