Download app.py from Rishe7/Databot: direct link, hf CLI and curl.
- Browser
- Download file 25.6 kB
-
https://huggingface.co/Rishe7/Databot/resolve/main/app.py
- Command line
-
hf download hf://Rishe7/Databot/app.py
-
curl -L -o app.py https://huggingface.co/Rishe7/Databot/resolve/main/app.py
25.6 kB
| import streamlit as st | |
| import pandas as pd | |
| import sqlalchemy as sql | |
| from langchain_groq import ChatGroq | |
| from langchain.prompts import PromptTemplate | |
| import streamlit.components.v1 as components | |
| from langchain_core.output_parsers import StrOutputParser | |
| import base64 | |
| import os | |
| from agents.sql_database_agent import SQLDatabaseAgent | |
| from agents.eda_database_agent import run_visualization_agent | |
| DB_PATH = "uploaded_csv_db.db" | |
| TABLE_NAME = "uploaded_data" | |
| # Page config | |
| st.set_page_config(page_icon="π", layout="wide", page_title="All-in-One Data Chatbot") | |
| st.title("π€ Ask Anything About Your Data") | |
| # Sidebar | |
| with st.sidebar: | |
| st.header("βοΈ Settings") | |
| uploaded_file = st.file_uploader("π Upload a CSV", type=["csv"]) | |
| st.markdown("---") | |
| st.header("π Groq API Key") | |
| groq_api_key = st.text_input("Enter your Groq API Key", type="password", key="groq_api_key") | |
| if groq_api_key: | |
| st.session_state["GROQ_API_KEY"] = groq_api_key | |
| else: | |
| groq_api_key = st.session_state.get("GROQ_API_KEY", None) | |
| # --- Ultimate Data Analytics Agent UI --- | |
| # Add sidebar agent/task selector | |
| with st.sidebar: | |
| st.header("π§° Agent/Task Selector") | |
| agent_task = st.radio( | |
| "Select task:", | |
| [ | |
| "Data Overview", | |
| "EDA", | |
| "Data Cleaning", | |
| "Visualization", | |
| "Query/Ask", | |
| "AutoML", | |
| "Advanced", | |
| "Report", | |
| "Data Quality", | |
| "Smart Cleaning", | |
| "Auto Insights", | |
| "Pipeline Builder", | |
| "Collaboration", | |
| "Alerts & Monitoring" | |
| ], | |
| index=0 | |
| ) | |
| # Define the prompt template | |
| decision_template = """ | |
| You are a smart assistant designed to choose the correct tool for analyzing a dataset. | |
| Your job is to decide which **agent** to activate based on the user's input. Analyze the question and return ONLY one of the following options: | |
| - Agent1 (π§ Data Exploration / Statistics): | |
| - When the user asks for high-level summaries, insights, dataset descriptions, distributions, or overall explanations. | |
| - Examples: "Describe this dataset", "What is the structure?", "Give me a summary", "What is interesting in the data?" | |
| - Agent2 (π Data Visualization): | |
| - When the user asks to visualize or plot the data (heatmap, histogram, pie chart, trends, etc). | |
| - Examples: "Show correlation heatmap", "Plot price trends", "Visualize monthly sales" | |
| - Agent3 (ποΈ SQL Query): | |
| - When the user is asking for precise calculations, aggregations, filtering, or row-level data that can be fetched using SQL queries. | |
| - Look for keywords like: "average", "total", "count", "maximum", "minimum", "select", "filter", "how many", "retrieve", or if dates/time ranges are involved. | |
| - Examples: "What was the average sales in March?", "Get rows where quantity > 100", "Show all orders from 2022" | |
| π Return only **Agent1**, **Agent2**, or **Agent3**. No explanations. | |
| User Input: {user_input} | |
| """ | |
| decision_maker = PromptTemplate(input_variables=["user_input"], template=decision_template) | |
| # Initialize LLM | |
| llm = ChatGroq( | |
| model="llama3-70b-8192", | |
| api_key=groq_api_key or os.getenv("TOKEN"), | |
| ) | |
| def upload_csv_to_sqlite(file, db_path: str, table_name: str, conn) -> pd.DataFrame: | |
| df.columns = [col.strip().replace(" ", "_") for col in df.columns] | |
| try: | |
| df.to_sql(table_name, conn, if_exists="replace", index=False) | |
| except Exception as e: | |
| raise RuntimeError(f"Failed to upload data to SQLite: {e}") | |
| return df | |
| def create_download_link(path, filename="sweetviz_report.html"): | |
| with open(path, "rb") as f: | |
| data = f.read() | |
| b64 = base64.b64encode(data).decode() | |
| href = f'<a href="data:text/html;base64,{b64}" download="{filename}">π₯ Download Sweetviz Report</a>' | |
| return href | |
| # If file uploaded | |
| if uploaded_file: | |
| # Load into SQLite | |
| df = pd.read_csv(uploaded_file) | |
| sql_engine = sql.create_engine("sqlite:///" + DB_PATH) | |
| conn = sql_engine.connect() | |
| df = upload_csv_to_sqlite(uploaded_file, DB_PATH, TABLE_NAME, conn) | |
| st.success("β CSV uploaded and loaded into SQLite!") | |
| st.dataframe(df.head(), use_container_width=True) | |
| st.session_state["data_raw"] = df.to_dict(orient="records") | |
| # Initialize agents | |
| sql_agent = SQLDatabaseAgent(llm=llm, conn=conn) | |
| # --- Data Quality Tab --- | |
| if agent_task == "Data Quality": | |
| st.subheader("Automated Data Quality Assessment") | |
| if uploaded_file: | |
| quality_issues = [] | |
| if df.isnull().sum().sum() > 0: | |
| quality_issues.append("Missing values detected.") | |
| print("[Data Quality] Missing values detected.") | |
| if df.duplicated().sum() > 0: | |
| quality_issues.append("Duplicate rows detected.") | |
| print("[Data Quality] Duplicate rows detected.") | |
| if not all([pd.api.types.is_numeric_dtype(df[col]) or pd.api.types.is_string_dtype(df[col]) for col in df.columns]): | |
| quality_issues.append("Inconsistent data types detected.") | |
| print("[Data Quality] Inconsistent data types detected.") | |
| health_score = 100-len(quality_issues)*30 | |
| st.write("**Data Health Score:**", health_score) | |
| print(f"[Data Quality] Data Health Score: {health_score}") | |
| st.write("**Issues:**") | |
| for issue in quality_issues: | |
| st.warning(issue) | |
| if not quality_issues: | |
| st.success("No major data quality issues detected!") | |
| print("[Data Quality] No major data quality issues detected!") | |
| else: | |
| st.info("Upload a CSV file to assess data quality.") | |
| print("[Data Quality] No file uploaded.") | |
| # --- Smart Cleaning Tab --- | |
| elif agent_task == "Smart Cleaning": | |
| st.subheader("Smart Data Cleaning") | |
| if uploaded_file: | |
| st.write("Preview and apply recommended cleaning steps:") | |
| if st.button("Drop duplicates"): | |
| df.drop_duplicates(inplace=True) | |
| st.success("Duplicates dropped.") | |
| print("[Smart Cleaning] Duplicates dropped.") | |
| if st.button("Fill missing values (0)"): | |
| df.fillna(0, inplace=True) | |
| st.success("Missing values filled with 0.") | |
| print("[Smart Cleaning] Missing values filled with 0.") | |
| st.dataframe(df.head(), use_container_width=True) | |
| else: | |
| st.info("Upload a CSV file to use smart cleaning.") | |
| print("[Smart Cleaning] No file uploaded.") | |
| # --- Auto Insights Tab --- | |
| elif agent_task == "Auto Insights": | |
| st.subheader("Automated Insights & Recommendations") | |
| if uploaded_file: | |
| st.write("Key trends and recommendations:") | |
| st.write("- Top correlated features:", df.corr(numeric_only=True).abs().sum().sort_values(ascending=False).head(3).to_dict()) | |
| st.write("- Row count:", len(df)) | |
| st.write("- Column count:", len(df.columns)) | |
| st.write("- Sample mean values:", df.mean(numeric_only=True).to_dict()) | |
| # Add more insights as needed | |
| else: | |
| st.info("Upload a CSV file to get insights.") | |
| # --- Pipeline Builder Tab --- | |
| elif agent_task == "Pipeline Builder": | |
| st.subheader("Data Pipeline Builder (Prototype)") | |
| st.write("Chain together cleaning, transformation, and modeling steps (coming soon!)") | |
| # --- Collaboration Tab --- | |
| elif agent_task == "Collaboration": | |
| st.subheader("Collaboration Tools") | |
| st.write("Share your analysis session or dashboard with others (coming soon!)") | |
| # --- Alerts & Monitoring Tab --- | |
| elif agent_task == "Alerts & Monitoring": | |
| st.subheader("Alerts & Monitoring") | |
| st.write("Set up alerts for data changes, anomalies, or model drift (coming soon!)") | |
| # Main area: show content based on selected task | |
| else: | |
| tabs = st.tabs([ | |
| "Data Overview", | |
| "EDA", | |
| "Data Cleaning", | |
| "Visualization", | |
| "Query/Ask", | |
| "AutoML", | |
| "Advanced", | |
| "Report" | |
| ]) | |
| with tabs[0]: | |
| st.subheader("Data Overview") | |
| st.dataframe(df.head(), use_container_width=True) | |
| st.write(df.info()) | |
| st.write(df.describe(include='all')) | |
| with tabs[1]: | |
| st.subheader("EDA") | |
| from ydata_profiling import ProfileReport | |
| import tempfile | |
| import os | |
| st.write("Generate and explore an automated EDA report for your data.") | |
| if st.button("Run EDA Report"): | |
| with st.spinner("Generating EDA report..."): | |
| profile = ProfileReport(df, title="Pandas Profiling Report", explorative=True) | |
| with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as tmpfile: | |
| profile.to_file(tmpfile.name) | |
| report_path = tmpfile.name | |
| st.success("EDA report generated!") | |
| with open(report_path, "r", encoding="utf-8") as f: | |
| html_content = f.read() | |
| components.html(html_content, height=800, scrolling=True) | |
| # Download link | |
| b64 = base64.b64encode(open(report_path, "rb").read()).decode() | |
| href = f'<a href="data:text/html;base64,{b64}" download="EDA_Report.html">π₯ Download EDA Report</a>' | |
| st.markdown(href, unsafe_allow_html=True) | |
| with tabs[2]: | |
| st.subheader("Data Cleaning") | |
| st.write("Interactively clean and transform your data.") | |
| # Show missing values | |
| st.markdown("#### Missing Values") | |
| missing = df.isnull().sum() | |
| st.dataframe(missing[missing > 0], use_container_width=True) | |
| # Drop missing values | |
| if st.button("Drop rows with missing values"): | |
| df.dropna(inplace=True) | |
| st.success("Rows with missing values dropped.") | |
| # Fill missing values | |
| fill_col = st.selectbox("Select column to fill missing values", [col for col in df.columns if df[col].isnull().sum() > 0], key="fill_col") | |
| fill_value = st.text_input("Fill missing with (value or 'mean', 'median', 'mode')", key="fill_val") | |
| if st.button("Fill missing values") and fill_col and fill_value: | |
| if fill_value.lower() == "mean": | |
| df[fill_col].fillna(df[fill_col].mean(), inplace=True) | |
| elif fill_value.lower() == "median": | |
| df[fill_col].fillna(df[fill_col].median(), inplace=True) | |
| elif fill_value.lower() == "mode": | |
| df[fill_col].fillna(df[fill_col].mode()[0], inplace=True) | |
| else: | |
| df[fill_col].fillna(fill_value, inplace=True) | |
| st.success(f"Filled missing values in {fill_col}.") | |
| # Remove duplicates | |
| if st.button("Remove duplicate rows"): | |
| df.drop_duplicates(inplace=True) | |
| st.success("Duplicate rows removed.") | |
| # Encode categoricals | |
| cat_cols = df.select_dtypes(include=['object', 'category']).columns.tolist() | |
| if cat_cols: | |
| enc_col = st.selectbox("Select categorical column to encode", cat_cols, key="enc_col") | |
| if st.button("One-hot encode column"): | |
| df = pd.get_dummies(df, columns=[enc_col]) | |
| st.success(f"One-hot encoded {enc_col}.") | |
| # Normalize columns | |
| num_cols = df.select_dtypes(include=['number']).columns.tolist() | |
| norm_col = st.selectbox("Select numeric column to normalize", num_cols, key="norm_col") | |
| if st.button("Normalize column"): | |
| df[norm_col] = (df[norm_col] - df[norm_col].mean()) / df[norm_col].std() | |
| st.success(f"Normalized {norm_col}.") | |
| st.dataframe(df.head(), use_container_width=True) | |
| with tabs[3]: | |
| st.subheader("Visualization") | |
| st.write("Create custom and automated visualizations.") | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| import plotly.express as px | |
| plot_types = ["Histogram", "Scatter", "Bar", "Line", "Box", "Pie"] | |
| plot_type = st.selectbox("Select plot type", plot_types, key="plot_type") | |
| x_col = st.selectbox("X axis", df.columns, key="x_col") | |
| y_col = st.selectbox("Y axis (optional)", [None] + list(df.columns), key="y_col") | |
| group_col = st.selectbox("Group/Color by (optional)", [None] + list(df.columns), key="group_col") | |
| if st.button("Generate Plot"): | |
| fig = None | |
| if plot_type == "Histogram": | |
| fig = px.histogram(df, x=x_col, color=group_col if group_col else None) | |
| elif plot_type == "Scatter" and y_col: | |
| fig = px.scatter(df, x=x_col, y=y_col, color=group_col if group_col else None) | |
| elif plot_type == "Bar" and y_col: | |
| fig = px.bar(df, x=x_col, y=y_col, color=group_col if group_col else None) | |
| elif plot_type == "Line" and y_col: | |
| fig = px.line(df, x=x_col, y=y_col, color=group_col if group_col else None) | |
| elif plot_type == "Box" and y_col: | |
| fig = px.box(df, x=x_col, y=y_col, color=group_col if group_col else None) | |
| elif plot_type == "Pie": | |
| fig = px.pie(df, names=x_col, values=y_col if y_col else None) | |
| if fig: | |
| st.plotly_chart(fig, use_container_width=True) | |
| st.success("Plot generated!") | |
| else: | |
| st.warning("Please select valid columns for the selected plot type.") | |
| with tabs[4]: | |
| st.subheader("Query/Ask") | |
| st.write("Chat with your data using natural language or SQL.") | |
| from streamlit_chat import message | |
| if "chat_history" not in st.session_state: | |
| st.session_state["chat_history"] = [] | |
| user_query = st.text_input("Enter your question or SQL query:") | |
| if st.button("Send") and user_query: | |
| st.session_state["chat_history"].append({"role": "user", "content": user_query}) | |
| try: | |
| if user_query.strip().lower().startswith("select"): | |
| # SQL mode | |
| result = pd.read_sql_query(user_query, conn) | |
| st.session_state["chat_history"].append({"role": "assistant", "content": result.to_markdown()}) | |
| else: | |
| # Natural language mode (LLM) | |
| decision_chain = decision_maker | llm | StrOutputParser() | |
| decision = decision_chain.invoke({"user_input": user_query}) | |
| if decision.strip() == "Agent3": | |
| sql_agent.smart_filter_schema(user_query) | |
| sql_query = sql_agent.generate_sql_query(user_query) | |
| result = sql_agent.run_sql_query(sql_query) | |
| sql_response = f"**π§ Generated SQL Query:**\n```sql\n{sql_query}\n```" | |
| if isinstance(result, pd.DataFrame): | |
| sql_response += f"\n**π Query Result:**\n{result.to_markdown()}" | |
| else: | |
| sql_response += f"\n{result}" | |
| st.session_state["chat_history"].append({"role": "assistant", "content": sql_response}) | |
| else: | |
| st.session_state["chat_history"].append({"role": "assistant", "content": "LLM: This would route to EDA or Visualization agent. (Stub)"}) | |
| except Exception as e: | |
| st.session_state["chat_history"].append({"role": "assistant", "content": f"Error: {e}"}) | |
| # Display chat | |
| for i, msg in enumerate(st.session_state["chat_history"]): | |
| is_user = msg["role"] == "user" | |
| message(msg["content"], is_user=is_user, key=f"chat_{i}") | |
| with tabs[5]: | |
| st.subheader("AutoML") | |
| st.write("Train an AutoML model on your data.") | |
| from pycaret.classification import setup as cls_setup, compare_models as cls_compare, predict_model as cls_predict | |
| from pycaret.regression import setup as reg_setup, compare_models as reg_compare, predict_model as reg_predict | |
| target_col = st.selectbox("Select target column", df.columns, key="target_col") | |
| # Feature selection: all columns except target by default | |
| default_features = [col for col in df.columns if col != target_col] | |
| selected_features = st.multiselect("Select features to include", options=[col for col in df.columns if col != target_col], default=default_features, key="feature_cols") | |
| use_all_features = st.checkbox("Use all features for ML", value=True) | |
| task_type = st.selectbox("Task type", ["Classification", "Regression"], key="task_type") | |
| if use_all_features: | |
| selected_features = [col for col in df.columns if col != target_col] | |
| if st.button("Run AutoML"): | |
| # Show progress and logs | |
| progress_bar = st.progress(0) | |
| log_area = st.empty() | |
| with st.spinner("Running AutoML..."): | |
| try: | |
| df_selected = df[selected_features + [target_col]] | |
| log_msgs = [] | |
| def log(msg, pct=None): | |
| log_msgs.append(msg) | |
| log_area.code("\n".join(log_msgs)) | |
| if pct is not None: | |
| progress_bar.progress(pct) | |
| log("Setting up experiment...", 10) | |
| if task_type == "Classification": | |
| s = cls_setup(df_selected, target=target_col, session_id=123, html=False, log_experiment=False) | |
| log("Comparing models...", 40) | |
| best_model = cls_compare() | |
| if isinstance(best_model, list): | |
| best_model = best_model[0] | |
| log("Predicting on data...", 70) | |
| preds = cls_predict(best_model, data=df_selected) | |
| log("Calculating metrics...", 90) | |
| st.success("Best classification model trained!") | |
| st.write(best_model) | |
| st.dataframe(preds, use_container_width=True) | |
| from pycaret.classification import pull | |
| results = pull() | |
| st.subheader("Model Comparison & Metrics") | |
| st.dataframe(results, use_container_width=True) | |
| from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report | |
| y_true = df_selected[target_col] | |
| y_pred = preds['Label'] if 'Label' in preds.columns else preds.iloc[:, -1] | |
| st.write(f"Accuracy: {accuracy_score(y_true, y_pred):.4f}") | |
| st.write(f"Precision: {precision_score(y_true, y_pred, average='weighted', zero_division=0):.4f}") | |
| st.write(f"Recall: {recall_score(y_true, y_pred, average='weighted', zero_division=0):.4f}") | |
| st.write(f"F1 Score: {f1_score(y_true, y_pred, average='weighted', zero_division=0):.4f}") | |
| st.text("\nClassification Report:\n" + classification_report(y_true, y_pred, zero_division=0)) | |
| log("Done!", 100) | |
| else: | |
| s = reg_setup(df_selected, target=target_col, session_id=123, html=False, log_experiment=False) | |
| log("Comparing models...", 40) | |
| best_model = reg_compare() | |
| if isinstance(best_model, list): | |
| best_model = best_model[0] | |
| log("Predicting on data...", 70) | |
| preds = reg_predict(best_model, data=df_selected) | |
| log("Calculating metrics...", 90) | |
| st.success("Best regression model trained!") | |
| st.write(best_model) | |
| st.dataframe(preds, use_container_width=True) | |
| from pycaret.regression import pull | |
| results = pull() | |
| st.subheader("Model Comparison & Metrics") | |
| st.dataframe(results, use_container_width=True) | |
| from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error | |
| y_true = df_selected[target_col] | |
| y_pred = preds['Label'] if 'Label' in preds.columns else preds.iloc[:, -1] | |
| st.write(f"R2 Score: {r2_score(y_true, y_pred):.4f}") | |
| st.write(f"MSE: {mean_squared_error(y_true, y_pred):.4f}") | |
| st.write(f"MAE: {mean_absolute_error(y_true, y_pred):.4f}") | |
| log("Done!", 100) | |
| except Exception as e: | |
| st.error(f"AutoML Error: {e}") | |
| log(f"Error: {e}", 100) | |
| with tabs[6]: | |
| st.subheader("Advanced") | |
| st.write("Time series forecasting, clustering, and anomaly detection coming soon!") | |
| # TODO: integrate advanced analytics agents | |
| with tabs[7]: | |
| st.subheader("Report") | |
| st.write("Generate and download a summary report of your analysis. (Coming soon!)") | |
| # TODO: integrate report agent | |
| # User input | |
| st.markdown("### π¬ Ask a question about your data:") | |
| user_question = st.chat_input("Ask anything...") | |
| if user_question: | |
| with st.chat_message("user"): | |
| st.markdown(user_question) | |
| decision_chain = decision_maker | llm | StrOutputParser() | |
| decision = decision_chain.invoke({ | |
| "user_input": user_question | |
| }) | |
| print("decision = ", decision) | |
| with st.chat_message("assistant"): | |
| with st.spinner("π€ Let me figure that out..."): | |
| try: | |
| # Use prompt to guide routing (lightweight classifier) | |
| if decision.strip() == "Agent1" or decision.strip() == "Agent2": | |
| print("right direction") | |
| # Go to visualization | |
| content, artifact = run_visualization_agent( | |
| conn=conn, | |
| question=user_question, | |
| llm=llm, | |
| data_raw=st.session_state["data_raw"], | |
| decision = decision | |
| ) | |
| st.markdown(content) | |
| #print("artifact = ", artifact.type) | |
| if artifact: | |
| if isinstance(artifact, list): | |
| for fig in artifact: | |
| st.plotly_chart(fig, use_container_width=True) | |
| elif isinstance(artifact, dict): | |
| # After you receive content and artifact from the agent | |
| report_path = artifact["report_path"] | |
| # Preview Sweetviz report | |
| with open(report_path, "r", encoding="utf-8") as f: | |
| html_content = f.read() | |
| components.html(html_content, height=800, scrolling=True) | |
| # Add download link | |
| download_link = create_download_link(report_path) | |
| st.markdown(download_link, unsafe_allow_html=True) | |
| else: | |
| st.error("β οΈ Invalid artifact format.") | |
| elif decision.strip() == "Agent3": | |
| print("sql direction") | |
| # Go to SQL | |
| sql_agent.smart_filter_schema(user_question) | |
| sql_query = sql_agent.generate_sql_query(user_question) | |
| result = sql_agent.run_sql_query(sql_query) | |
| st.markdown("**π§ Generated SQL Query:**") | |
| st.code(sql_query, language="sql") | |
| if isinstance(result, pd.DataFrame): | |
| st.markdown("**π Query Result:**") | |
| st.dataframe(result, use_container_width=True) | |
| else: | |
| st.warning(result) | |
| except Exception as e: | |
| st.error(f"β Error: {e}") | |