Rishe7 commited on
Commit
e789e15
·
verified ·
1 Parent(s): 7bf6685

Upload 6 files

Browse files
Files changed (6) hide show
  1. automl.py +4 -0
  2. cleaning.py +4 -0
  3. dataframe.py +59 -0
  4. eda.py +57 -0
  5. python_run.py +81 -0
  6. sql.py +85 -0
automl.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Tools for AutoML (scaffold)
2
+ def automl_tool(data, target_col, task_type):
3
+ # Placeholder for AutoML utility functions
4
+ pass
cleaning.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # Tools for data cleaning (scaffold)
2
+ def cleaning_tool(data):
3
+ # Placeholder for cleaning utilities
4
+ pass
dataframe.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ from typing import Union, List, Dict
3
+
4
+ def get_dataframe_summary(
5
+ dataframes: Union[pd.DataFrame, List[pd.DataFrame], Dict[str, pd.DataFrame]],
6
+ n_sample: int = 10,
7
+ skip_stats: bool = False,
8
+ ) -> List[Dict]:
9
+ """
10
+ Return a structured summary for one or more DataFrames. Each summary includes:
11
+ - Dataset name
12
+ - Shape (rows, columns)
13
+ - Column data types
14
+ - Missing value percentage per column
15
+ - Unique value count per column
16
+ - Descriptive statistics (optional)
17
+ """
18
+
19
+ summaries = []
20
+
21
+ if isinstance(dataframes, dict):
22
+ for dataset_name, df in dataframes.items():
23
+ summaries.append(_summarize_dataframe(df, dataset_name, skip_stats))
24
+ elif isinstance(dataframes, pd.DataFrame):
25
+ summaries.append(_summarize_dataframe(dataframes, "Single_Dataset", skip_stats))
26
+ elif isinstance(dataframes, list):
27
+ for idx, df in enumerate(dataframes):
28
+ dataset_name = f"Dataset_{idx}"
29
+ summaries.append(_summarize_dataframe(df, dataset_name, skip_stats))
30
+ else:
31
+ raise TypeError("Input must be a DataFrame, list, or dictionary of DataFrames.")
32
+
33
+ return summaries
34
+
35
+ def _summarize_dataframe(df: pd.DataFrame, dataset_name: str, skip_stats: bool) -> Dict:
36
+ df = df.copy()
37
+ df = df.apply(lambda col: col.map(lambda x: str(x) if isinstance(x, dict) else x))
38
+
39
+ numeric_cols = df.select_dtypes(include='number').columns.tolist()
40
+ column_types = df.dtypes.astype(str).tolist()
41
+ columns = df.columns.tolist()
42
+
43
+ summary = {
44
+ "dataset_name": dataset_name,
45
+ "shape": {"rows": df.shape[0], "columns": df.shape[1]},
46
+ "columns": columns,
47
+ "column_types": column_types,
48
+ "missing_percent": df.isna().mean().round(4).mul(100).tolist(),
49
+ "unique_value_counts": df.nunique().tolist()
50
+ }
51
+
52
+ if not skip_stats:
53
+ desc = df[numeric_cols].describe().loc[['count', 'mean', '50%', 'min', 'max']].round(2)
54
+ summary["describe"] = {
55
+ stat: desc.loc[stat].tolist()
56
+ for stat in desc.index
57
+ }
58
+
59
+ return summary
eda.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import sweetviz as sv
3
+ from tools.dataframe import get_dataframe_summary
4
+ from typing import Tuple, Dict
5
+ import os
6
+ import tempfile
7
+
8
+
9
+ def explain_data(data_raw, n_sample= 10) -> str:
10
+ """
11
+ Tool: explain_data
12
+ Description:
13
+ Provides an extensive, narrative summary of a DataFrame including its shape, column types,
14
+ missing value percentages, unique counts, sample rows, and (if not skipped) descriptive stats/info.
15
+
16
+ Parameters:
17
+ data_raw (dict): Raw data.
18
+ n_sample (int, default=30): Number of rows to display.
19
+ skip_stats (bool, default=False): If True, omit descriptive stats/info.
20
+
21
+ LLM Guidance:
22
+ Use when a detailed, human-readable explanation is needed—i.e., a full overview is preferred over a concise numerical summary.
23
+
24
+ Returns:
25
+ str: Detailed DataFrame summary.
26
+ """
27
+ print(" * Tool: explain_data")
28
+ import pandas as pd
29
+
30
+ result = get_dataframe_summary(
31
+ pd.DataFrame(data_raw), n_sample=10, skip_stats=False
32
+ )
33
+
34
+ return result
35
+
36
+
37
+
38
+ def generate_sweetviz_report(
39
+ data_raw: dict,
40
+ target: str = None,
41
+ report_name: str = "Report.html",
42
+ report_directory: str = None,
43
+ open_browser: bool = False,
44
+ ) -> Tuple[str, Dict]:
45
+ df = pd.DataFrame(data_raw)
46
+ analysis = sv.analyze(df, target_feat=target)
47
+
48
+ if report_directory is None:
49
+ report_directory = tempfile.mkdtemp()
50
+ report_path = os.path.join(report_directory, report_name)
51
+
52
+ analysis.show_html(filepath=report_path, open_browser=open_browser)
53
+
54
+ return (
55
+ f"Press on the download report button at the bottom",
56
+ {"report_path": report_path},
57
+ )
python_run.py ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import runpy
3
+ import os
4
+ import importlib.util
5
+ import subprocess
6
+ import sys
7
+
8
+
9
+ def install_missing_libraries(code):
10
+ """
11
+ Parses the Python code to identify missing libraries and installs them.
12
+ """
13
+ # Extract import statements from the code
14
+ import_statements = re.findall(r"^\s*(?:import|from)\s+([a-zA-Z0-9_\.]+)", code, re.MULTILINE)
15
+ missing_libraries = []
16
+
17
+ for module in import_statements:
18
+ # Check if the module is installed
19
+ try:
20
+ importlib.import_module(module.split('.')[0]) # Handle submodules like `pandas.core`
21
+ except ImportError:
22
+ missing_libraries.append(module.split('.')[0])
23
+
24
+ # Install missing libraries
25
+ if missing_libraries:
26
+ for lib in set(missing_libraries): # Remove duplicates
27
+ try:
28
+ print(f"Installing missing library: {lib}")
29
+ subprocess.check_call([sys.executable, "-m", "pip", "install", lib])
30
+ except Exception as e:
31
+ print(f"Failed to install {lib}: {e}")
32
+ else:
33
+ print("All required libraries are already installed.")
34
+
35
+
36
+ def extract_python_code(response):
37
+ """
38
+ Extracts the Python code block from the LLM response.
39
+ Assumes the code is enclosed within triple backticks (```).
40
+ """
41
+ code_block = re.search(r"```python\n(.*?)```", response, re.DOTALL)
42
+ code_block1 = re.search(r"```\n(.*?)```", response, re.DOTALL)
43
+ if code_block:
44
+ return code_block.group(1)
45
+ elif code_block1:
46
+ return code_block1.group(1)
47
+ else:
48
+ raise ValueError("No Python code block found in the response.")
49
+
50
+
51
+ def execute_visualizations(code, temp_file="temp_visualization.py"):
52
+ """
53
+ Executes the extracted Python code in a separate file and
54
+ returns the list of visualizations (figures).
55
+ """
56
+ try:
57
+ # Check for missing libraries and install them
58
+ install_missing_libraries(code)
59
+
60
+ # Write the code to a temporary file
61
+ with open(temp_file, "w") as file:
62
+ file.write(code)
63
+
64
+ # Dynamically load the module
65
+ spec = importlib.util.spec_from_file_location("temp_visualization", temp_file)
66
+ temp_module = importlib.util.module_from_spec(spec)
67
+ spec.loader.exec_module(temp_module)
68
+
69
+ # Call the create_visualizations function
70
+ if hasattr(temp_module, "create_visualizations"):
71
+ return temp_module.create_visualizations()
72
+ else:
73
+ raise AttributeError("No function 'create_visualizations' found in the provided code.")
74
+
75
+ except Exception as e:
76
+ return f"Error executing the code: {e}"
77
+
78
+ finally:
79
+ # Clean up the temporary file
80
+ if os.path.exists(temp_file):
81
+ os.remove(temp_file)
sql.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import sqlalchemy as sql
3
+ from sqlalchemy import inspect
4
+
5
+
6
+ def get_db_metadata(connection, n_samples=3) -> dict:
7
+ """
8
+ Collects metadata and sample data from a database, with safe identifier quoting and
9
+ basic dialect-aware row limiting. Prevents issues with spaces/reserved words in identifiers.
10
+
11
+ Parameters:
12
+ ----------
13
+ connection : Union[sql.engine.base.Connection, sql.engine.base.Engine]
14
+ An active SQLAlchemy connection or engine.
15
+ n_samples : int
16
+ Number of sample values to retrieve for each column.
17
+
18
+ Returns:
19
+ -------
20
+ dict
21
+ A dictionary with database metadata, including some sample data from each column.
22
+ """
23
+ is_engine = isinstance(connection, sql.engine.base.Engine)
24
+ conn = connection.connect() if is_engine else connection
25
+
26
+ metadata = {
27
+ "dialect": None,
28
+ "driver": None,
29
+ "connection_url": None,
30
+ "schemas": [],
31
+ }
32
+
33
+ try:
34
+ sql_engine = conn.engine
35
+ metadata["dialect"] = sql_engine.dialect.name
36
+ metadata["driver"] = sql_engine.driver
37
+ metadata["connection_url"] = str(sql_engine.url)
38
+
39
+ inspector = inspect(sql_engine)
40
+ preparer = inspector.bind.dialect.identifier_preparer
41
+
42
+ for schema_name in inspector.get_schema_names():
43
+ schema_obj = {"schema_name": schema_name, "tables": []}
44
+ tables = inspector.get_table_names(schema=schema_name)
45
+ for table_name in tables:
46
+ table_info = {"table_name": table_name, "columns": [], "primary_key": [], "foreign_keys": [], "indexes": []}
47
+
48
+ columns = inspector.get_columns(table_name, schema=schema_name)
49
+ for col in columns:
50
+ col_name = col["name"]
51
+ col_type = str(col["type"])
52
+ table_name_quoted = f"{preparer.quote_identifier(schema_name)}.{preparer.quote_identifier(table_name)}"
53
+ col_name_quoted = preparer.quote_identifier(col_name)
54
+ query = build_query(col_name_quoted, table_name_quoted, n_samples, metadata["dialect"])
55
+
56
+ try:
57
+ df = pd.read_sql(query, conn)
58
+ samples = df[col_name].head(n_samples).tolist()
59
+ except Exception as e:
60
+ samples = [f"Error retrieving data: {str(e)}"]
61
+
62
+ table_info["columns"].append({"name": col_name, "type": col_type, "sample_values": samples})
63
+
64
+ pk_constraint = inspector.get_pk_constraint(table_name, schema=schema_name)
65
+ table_info["primary_key"] = pk_constraint.get("constrained_columns", [])
66
+
67
+ fks = inspector.get_foreign_keys(table_name, schema=schema_name)
68
+ table_info["foreign_keys"] = [{"local_cols": fk["constrained_columns"], "referred_table": fk["referred_table"], "referred_cols": fk["referred_columns"]} for fk in fks]
69
+
70
+ idxs = inspector.get_indexes(table_name, schema=schema_name)
71
+ table_info["indexes"] = idxs
72
+
73
+ schema_obj["tables"].append(table_info)
74
+
75
+ metadata["schemas"].append(schema_obj)
76
+ finally:
77
+ if is_engine:
78
+ conn.close()
79
+
80
+ return metadata
81
+
82
+ def build_query(col_name_quoted: str, table_name_quoted: str, n: int, dialect_name: str) -> str:
83
+ if "sqlite" in dialect_name:
84
+ return f"SELECT {col_name_quoted} FROM {table_name_quoted} ORDER BY RANDOM() LIMIT {n}"
85
+ return f"SELECT {col_name_quoted} FROM {table_name_quoted} WHERE ROWNUM <= {n}"