Spaces:
Sleeping
Sleeping
| """Schema inference and column statistics.""" | |
| from __future__ import annotations | |
| from typing import Any | |
| import duckdb | |
| import polars as pl | |
| from visual_notebook.data.types import ColumnType, classify_column | |
| def infer_schema_duckdb(con: duckdb.DuckDBPyConnection, table_name: str) -> list[dict[str, Any]]: | |
| """Infer schema and statistics using DuckDB SUMMARIZE.""" | |
| try: | |
| rows = con.execute(f'SUMMARIZE "{table_name}"').fetchall() | |
| # SUMMARIZE returns: column_name, column_type, min, max, approx_unique, avg, std, q25, q50, q75, count, null_percentage | |
| columns = [] | |
| col_names = [d[0] for d in con.execute(f'SUMMARIZE "{table_name}"').description] | |
| total_rows = con.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone()[0] | |
| for row in rows: | |
| col = dict(zip(col_names, row)) | |
| col_name = col.get("column_name", col.get(col_names[0], "")) | |
| col_type = str(col.get("column_type", "")) | |
| null_pct = col.get("null_percentage", 0.0) | |
| null_count = int((null_pct / 100.0) * total_rows) if null_pct else 0 | |
| unique_count = col.get("approx_unique", 0) or 0 | |
| classified = classify_column(col_type, col_name, null_count, unique_count, total_rows) | |
| # Determine if numeric and compute median/iqr | |
| is_numeric = classified in (ColumnType.NUMERIC, ColumnType.GEO_LAT_LON) | |
| q50_val = col.get("q50") | |
| q25_val = col.get("q25") | |
| q75_val = col.get("q75") | |
| median_val = None | |
| iqr_val = None | |
| if is_numeric and q50_val is not None: | |
| median_val = float(q50_val) | |
| if is_numeric and q25_val is not None and q75_val is not None: | |
| iqr_val = float(q75_val) - float(q25_val) | |
| columns.append({ | |
| "name": col_name, | |
| "dtype": col_type, | |
| "column_type": classified.value, | |
| "null_count": null_count, | |
| "unique_count": unique_count, | |
| "min": col.get("min"), | |
| "max": col.get("max"), | |
| "mean": col.get("avg"), | |
| "std": col.get("std"), | |
| "q25": col.get("q25"), | |
| "q50": col.get("q50"), | |
| "q75": col.get("q75"), | |
| "count": col.get("count", total_rows), | |
| "has_numeric_stats": is_numeric, | |
| "median": median_val, | |
| "iqr": iqr_val, | |
| }) | |
| return columns | |
| except Exception: | |
| # Fallback: basic column info | |
| col_rows = con.execute( | |
| f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{table_name}'" | |
| ).fetchall() | |
| total_rows = con.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone()[0] | |
| columns = [] | |
| for name, dtype in col_rows: | |
| classified = classify_column(dtype, name, 0, 0, total_rows) | |
| is_numeric = classified in (ColumnType.NUMERIC, ColumnType.GEO_LAT_LON) | |
| columns.append({ | |
| "name": name, | |
| "dtype": dtype, | |
| "column_type": classified.value, | |
| "null_count": 0, | |
| "unique_count": 0, | |
| "min": None, | |
| "max": None, | |
| "mean": None, | |
| "std": None, | |
| "q25": None, | |
| "q50": None, | |
| "q75": None, | |
| "count": total_rows, | |
| "has_numeric_stats": is_numeric, | |
| "median": None, | |
| "iqr": None, | |
| }) | |
| return columns | |
| def column_stats_polars(df: pl.DataFrame, col_name: str) -> dict[str, Any]: | |
| """Compute detailed stats for a column using Polars.""" | |
| col = df[col_name] | |
| classified = classify_column(str(col.dtype), col_name, col.null_count(), col.n_unique(), df.height) | |
| is_numeric = classified in (ColumnType.NUMERIC, ColumnType.GEO_LAT_LON) | |
| stats = { | |
| "name": col_name, | |
| "dtype": str(col.dtype), | |
| "null_count": col.null_count(), | |
| "unique_count": col.n_unique(), | |
| "has_numeric_stats": is_numeric, | |
| "median": None, | |
| "iqr": None, | |
| } | |
| if col.dtype.is_numeric(): | |
| desc = col.describe() | |
| q25 = desc.filter(pl.col("statistic") == "25%")["value"][0] | |
| q50 = desc.filter(pl.col("statistic") == "50%")["value"][0] | |
| q75 = desc.filter(pl.col("statistic") == "75%")["value"][0] | |
| stats.update({ | |
| "mean": desc.filter(pl.col("statistic") == "mean")["value"][0], | |
| "std": desc.filter(pl.col("statistic") == "std")["value"][0], | |
| "min": desc.filter(pl.col("statistic") == "min")["value"][0], | |
| "max": desc.filter(pl.col("statistic") == "max")["value"][0], | |
| "q25": q25, | |
| "q50": q50, | |
| "q75": q75, | |
| "median": float(q50) if q50 is not None else None, | |
| "iqr": float(q75) - float(q25) if q25 is not None and q75 is not None else None, | |
| }) | |
| return stats | |