Upload 6 files
Browse files- automl.py +4 -0
- cleaning.py +4 -0
- dataframe.py +59 -0
- eda.py +57 -0
- python_run.py +81 -0
- sql.py +85 -0
automl.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Tools for AutoML (scaffold)
|
| 2 |
+
def automl_tool(data, target_col, task_type):
|
| 3 |
+
# Placeholder for AutoML utility functions
|
| 4 |
+
pass
|
cleaning.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Tools for data cleaning (scaffold)
|
| 2 |
+
def cleaning_tool(data):
|
| 3 |
+
# Placeholder for cleaning utilities
|
| 4 |
+
pass
|
dataframe.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pandas as pd
|
| 2 |
+
from typing import Union, List, Dict
|
| 3 |
+
|
| 4 |
+
def get_dataframe_summary(
|
| 5 |
+
dataframes: Union[pd.DataFrame, List[pd.DataFrame], Dict[str, pd.DataFrame]],
|
| 6 |
+
n_sample: int = 10,
|
| 7 |
+
skip_stats: bool = False,
|
| 8 |
+
) -> List[Dict]:
|
| 9 |
+
"""
|
| 10 |
+
Return a structured summary for one or more DataFrames. Each summary includes:
|
| 11 |
+
- Dataset name
|
| 12 |
+
- Shape (rows, columns)
|
| 13 |
+
- Column data types
|
| 14 |
+
- Missing value percentage per column
|
| 15 |
+
- Unique value count per column
|
| 16 |
+
- Descriptive statistics (optional)
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
summaries = []
|
| 20 |
+
|
| 21 |
+
if isinstance(dataframes, dict):
|
| 22 |
+
for dataset_name, df in dataframes.items():
|
| 23 |
+
summaries.append(_summarize_dataframe(df, dataset_name, skip_stats))
|
| 24 |
+
elif isinstance(dataframes, pd.DataFrame):
|
| 25 |
+
summaries.append(_summarize_dataframe(dataframes, "Single_Dataset", skip_stats))
|
| 26 |
+
elif isinstance(dataframes, list):
|
| 27 |
+
for idx, df in enumerate(dataframes):
|
| 28 |
+
dataset_name = f"Dataset_{idx}"
|
| 29 |
+
summaries.append(_summarize_dataframe(df, dataset_name, skip_stats))
|
| 30 |
+
else:
|
| 31 |
+
raise TypeError("Input must be a DataFrame, list, or dictionary of DataFrames.")
|
| 32 |
+
|
| 33 |
+
return summaries
|
| 34 |
+
|
| 35 |
+
def _summarize_dataframe(df: pd.DataFrame, dataset_name: str, skip_stats: bool) -> Dict:
|
| 36 |
+
df = df.copy()
|
| 37 |
+
df = df.apply(lambda col: col.map(lambda x: str(x) if isinstance(x, dict) else x))
|
| 38 |
+
|
| 39 |
+
numeric_cols = df.select_dtypes(include='number').columns.tolist()
|
| 40 |
+
column_types = df.dtypes.astype(str).tolist()
|
| 41 |
+
columns = df.columns.tolist()
|
| 42 |
+
|
| 43 |
+
summary = {
|
| 44 |
+
"dataset_name": dataset_name,
|
| 45 |
+
"shape": {"rows": df.shape[0], "columns": df.shape[1]},
|
| 46 |
+
"columns": columns,
|
| 47 |
+
"column_types": column_types,
|
| 48 |
+
"missing_percent": df.isna().mean().round(4).mul(100).tolist(),
|
| 49 |
+
"unique_value_counts": df.nunique().tolist()
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
if not skip_stats:
|
| 53 |
+
desc = df[numeric_cols].describe().loc[['count', 'mean', '50%', 'min', 'max']].round(2)
|
| 54 |
+
summary["describe"] = {
|
| 55 |
+
stat: desc.loc[stat].tolist()
|
| 56 |
+
for stat in desc.index
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
return summary
|
eda.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pandas as pd
|
| 2 |
+
import sweetviz as sv
|
| 3 |
+
from tools.dataframe import get_dataframe_summary
|
| 4 |
+
from typing import Tuple, Dict
|
| 5 |
+
import os
|
| 6 |
+
import tempfile
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def explain_data(data_raw, n_sample= 10) -> str:
|
| 10 |
+
"""
|
| 11 |
+
Tool: explain_data
|
| 12 |
+
Description:
|
| 13 |
+
Provides an extensive, narrative summary of a DataFrame including its shape, column types,
|
| 14 |
+
missing value percentages, unique counts, sample rows, and (if not skipped) descriptive stats/info.
|
| 15 |
+
|
| 16 |
+
Parameters:
|
| 17 |
+
data_raw (dict): Raw data.
|
| 18 |
+
n_sample (int, default=30): Number of rows to display.
|
| 19 |
+
skip_stats (bool, default=False): If True, omit descriptive stats/info.
|
| 20 |
+
|
| 21 |
+
LLM Guidance:
|
| 22 |
+
Use when a detailed, human-readable explanation is needed—i.e., a full overview is preferred over a concise numerical summary.
|
| 23 |
+
|
| 24 |
+
Returns:
|
| 25 |
+
str: Detailed DataFrame summary.
|
| 26 |
+
"""
|
| 27 |
+
print(" * Tool: explain_data")
|
| 28 |
+
import pandas as pd
|
| 29 |
+
|
| 30 |
+
result = get_dataframe_summary(
|
| 31 |
+
pd.DataFrame(data_raw), n_sample=10, skip_stats=False
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
return result
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def generate_sweetviz_report(
|
| 39 |
+
data_raw: dict,
|
| 40 |
+
target: str = None,
|
| 41 |
+
report_name: str = "Report.html",
|
| 42 |
+
report_directory: str = None,
|
| 43 |
+
open_browser: bool = False,
|
| 44 |
+
) -> Tuple[str, Dict]:
|
| 45 |
+
df = pd.DataFrame(data_raw)
|
| 46 |
+
analysis = sv.analyze(df, target_feat=target)
|
| 47 |
+
|
| 48 |
+
if report_directory is None:
|
| 49 |
+
report_directory = tempfile.mkdtemp()
|
| 50 |
+
report_path = os.path.join(report_directory, report_name)
|
| 51 |
+
|
| 52 |
+
analysis.show_html(filepath=report_path, open_browser=open_browser)
|
| 53 |
+
|
| 54 |
+
return (
|
| 55 |
+
f"Press on the download report button at the bottom",
|
| 56 |
+
{"report_path": report_path},
|
| 57 |
+
)
|
python_run.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
import runpy
|
| 3 |
+
import os
|
| 4 |
+
import importlib.util
|
| 5 |
+
import subprocess
|
| 6 |
+
import sys
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def install_missing_libraries(code):
|
| 10 |
+
"""
|
| 11 |
+
Parses the Python code to identify missing libraries and installs them.
|
| 12 |
+
"""
|
| 13 |
+
# Extract import statements from the code
|
| 14 |
+
import_statements = re.findall(r"^\s*(?:import|from)\s+([a-zA-Z0-9_\.]+)", code, re.MULTILINE)
|
| 15 |
+
missing_libraries = []
|
| 16 |
+
|
| 17 |
+
for module in import_statements:
|
| 18 |
+
# Check if the module is installed
|
| 19 |
+
try:
|
| 20 |
+
importlib.import_module(module.split('.')[0]) # Handle submodules like `pandas.core`
|
| 21 |
+
except ImportError:
|
| 22 |
+
missing_libraries.append(module.split('.')[0])
|
| 23 |
+
|
| 24 |
+
# Install missing libraries
|
| 25 |
+
if missing_libraries:
|
| 26 |
+
for lib in set(missing_libraries): # Remove duplicates
|
| 27 |
+
try:
|
| 28 |
+
print(f"Installing missing library: {lib}")
|
| 29 |
+
subprocess.check_call([sys.executable, "-m", "pip", "install", lib])
|
| 30 |
+
except Exception as e:
|
| 31 |
+
print(f"Failed to install {lib}: {e}")
|
| 32 |
+
else:
|
| 33 |
+
print("All required libraries are already installed.")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def extract_python_code(response):
|
| 37 |
+
"""
|
| 38 |
+
Extracts the Python code block from the LLM response.
|
| 39 |
+
Assumes the code is enclosed within triple backticks (```).
|
| 40 |
+
"""
|
| 41 |
+
code_block = re.search(r"```python\n(.*?)```", response, re.DOTALL)
|
| 42 |
+
code_block1 = re.search(r"```\n(.*?)```", response, re.DOTALL)
|
| 43 |
+
if code_block:
|
| 44 |
+
return code_block.group(1)
|
| 45 |
+
elif code_block1:
|
| 46 |
+
return code_block1.group(1)
|
| 47 |
+
else:
|
| 48 |
+
raise ValueError("No Python code block found in the response.")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def execute_visualizations(code, temp_file="temp_visualization.py"):
|
| 52 |
+
"""
|
| 53 |
+
Executes the extracted Python code in a separate file and
|
| 54 |
+
returns the list of visualizations (figures).
|
| 55 |
+
"""
|
| 56 |
+
try:
|
| 57 |
+
# Check for missing libraries and install them
|
| 58 |
+
install_missing_libraries(code)
|
| 59 |
+
|
| 60 |
+
# Write the code to a temporary file
|
| 61 |
+
with open(temp_file, "w") as file:
|
| 62 |
+
file.write(code)
|
| 63 |
+
|
| 64 |
+
# Dynamically load the module
|
| 65 |
+
spec = importlib.util.spec_from_file_location("temp_visualization", temp_file)
|
| 66 |
+
temp_module = importlib.util.module_from_spec(spec)
|
| 67 |
+
spec.loader.exec_module(temp_module)
|
| 68 |
+
|
| 69 |
+
# Call the create_visualizations function
|
| 70 |
+
if hasattr(temp_module, "create_visualizations"):
|
| 71 |
+
return temp_module.create_visualizations()
|
| 72 |
+
else:
|
| 73 |
+
raise AttributeError("No function 'create_visualizations' found in the provided code.")
|
| 74 |
+
|
| 75 |
+
except Exception as e:
|
| 76 |
+
return f"Error executing the code: {e}"
|
| 77 |
+
|
| 78 |
+
finally:
|
| 79 |
+
# Clean up the temporary file
|
| 80 |
+
if os.path.exists(temp_file):
|
| 81 |
+
os.remove(temp_file)
|
sql.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pandas as pd
|
| 2 |
+
import sqlalchemy as sql
|
| 3 |
+
from sqlalchemy import inspect
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def get_db_metadata(connection, n_samples=3) -> dict:
|
| 7 |
+
"""
|
| 8 |
+
Collects metadata and sample data from a database, with safe identifier quoting and
|
| 9 |
+
basic dialect-aware row limiting. Prevents issues with spaces/reserved words in identifiers.
|
| 10 |
+
|
| 11 |
+
Parameters:
|
| 12 |
+
----------
|
| 13 |
+
connection : Union[sql.engine.base.Connection, sql.engine.base.Engine]
|
| 14 |
+
An active SQLAlchemy connection or engine.
|
| 15 |
+
n_samples : int
|
| 16 |
+
Number of sample values to retrieve for each column.
|
| 17 |
+
|
| 18 |
+
Returns:
|
| 19 |
+
-------
|
| 20 |
+
dict
|
| 21 |
+
A dictionary with database metadata, including some sample data from each column.
|
| 22 |
+
"""
|
| 23 |
+
is_engine = isinstance(connection, sql.engine.base.Engine)
|
| 24 |
+
conn = connection.connect() if is_engine else connection
|
| 25 |
+
|
| 26 |
+
metadata = {
|
| 27 |
+
"dialect": None,
|
| 28 |
+
"driver": None,
|
| 29 |
+
"connection_url": None,
|
| 30 |
+
"schemas": [],
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
try:
|
| 34 |
+
sql_engine = conn.engine
|
| 35 |
+
metadata["dialect"] = sql_engine.dialect.name
|
| 36 |
+
metadata["driver"] = sql_engine.driver
|
| 37 |
+
metadata["connection_url"] = str(sql_engine.url)
|
| 38 |
+
|
| 39 |
+
inspector = inspect(sql_engine)
|
| 40 |
+
preparer = inspector.bind.dialect.identifier_preparer
|
| 41 |
+
|
| 42 |
+
for schema_name in inspector.get_schema_names():
|
| 43 |
+
schema_obj = {"schema_name": schema_name, "tables": []}
|
| 44 |
+
tables = inspector.get_table_names(schema=schema_name)
|
| 45 |
+
for table_name in tables:
|
| 46 |
+
table_info = {"table_name": table_name, "columns": [], "primary_key": [], "foreign_keys": [], "indexes": []}
|
| 47 |
+
|
| 48 |
+
columns = inspector.get_columns(table_name, schema=schema_name)
|
| 49 |
+
for col in columns:
|
| 50 |
+
col_name = col["name"]
|
| 51 |
+
col_type = str(col["type"])
|
| 52 |
+
table_name_quoted = f"{preparer.quote_identifier(schema_name)}.{preparer.quote_identifier(table_name)}"
|
| 53 |
+
col_name_quoted = preparer.quote_identifier(col_name)
|
| 54 |
+
query = build_query(col_name_quoted, table_name_quoted, n_samples, metadata["dialect"])
|
| 55 |
+
|
| 56 |
+
try:
|
| 57 |
+
df = pd.read_sql(query, conn)
|
| 58 |
+
samples = df[col_name].head(n_samples).tolist()
|
| 59 |
+
except Exception as e:
|
| 60 |
+
samples = [f"Error retrieving data: {str(e)}"]
|
| 61 |
+
|
| 62 |
+
table_info["columns"].append({"name": col_name, "type": col_type, "sample_values": samples})
|
| 63 |
+
|
| 64 |
+
pk_constraint = inspector.get_pk_constraint(table_name, schema=schema_name)
|
| 65 |
+
table_info["primary_key"] = pk_constraint.get("constrained_columns", [])
|
| 66 |
+
|
| 67 |
+
fks = inspector.get_foreign_keys(table_name, schema=schema_name)
|
| 68 |
+
table_info["foreign_keys"] = [{"local_cols": fk["constrained_columns"], "referred_table": fk["referred_table"], "referred_cols": fk["referred_columns"]} for fk in fks]
|
| 69 |
+
|
| 70 |
+
idxs = inspector.get_indexes(table_name, schema=schema_name)
|
| 71 |
+
table_info["indexes"] = idxs
|
| 72 |
+
|
| 73 |
+
schema_obj["tables"].append(table_info)
|
| 74 |
+
|
| 75 |
+
metadata["schemas"].append(schema_obj)
|
| 76 |
+
finally:
|
| 77 |
+
if is_engine:
|
| 78 |
+
conn.close()
|
| 79 |
+
|
| 80 |
+
return metadata
|
| 81 |
+
|
| 82 |
+
def build_query(col_name_quoted: str, table_name_quoted: str, n: int, dialect_name: str) -> str:
|
| 83 |
+
if "sqlite" in dialect_name:
|
| 84 |
+
return f"SELECT {col_name_quoted} FROM {table_name_quoted} ORDER BY RANDOM() LIMIT {n}"
|
| 85 |
+
return f"SELECT {col_name_quoted} FROM {table_name_quoted} WHERE ROWNUM <= {n}"
|