Spaces:
Sleeping
Sleeping
Deploy backend Docker Space
Browse files- .env.example +12 -0
- Dockerfile +23 -0
- README.md +177 -10
- pyproject.toml +46 -0
- src/autodata_agent/__init__.py +6 -0
- src/autodata_agent/api/app.py +71 -0
- src/autodata_agent/api/dependencies.py +41 -0
- src/autodata_agent/api/routes.py +111 -0
- src/autodata_agent/core/config.py +60 -0
- src/autodata_agent/core/errors.py +34 -0
- src/autodata_agent/core/json_utils.py +41 -0
- src/autodata_agent/core/schemas.py +211 -0
- src/autodata_agent/main.py +12 -0
- src/autodata_agent/services/analysis.py +298 -0
- src/autodata_agent/services/charts.py +151 -0
- src/autodata_agent/services/datasets.py +181 -0
- src/autodata_agent/services/executor.py +207 -0
- src/autodata_agent/services/llm.py +90 -0
- src/autodata_agent/services/profiling.py +127 -0
- src/autodata_agent/storage/session_store.py +116 -0
.env.example
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
AUTODATA_ENV=development
|
| 2 |
+
AUTODATA_STORAGE_DIR=data/runtime
|
| 3 |
+
AUTODATA_UPLOAD_DIR=data/uploads
|
| 4 |
+
AUTODATA_OLLAMA_BASE_URL=https://ollama.com
|
| 5 |
+
AUTODATA_OLLAMA_API_KEY=replace-with-your-ollama-api-key
|
| 6 |
+
AUTODATA_OLLAMA_MODEL=qwen3-coder:480b
|
| 7 |
+
AUTODATA_OLLAMA_THINK=false
|
| 8 |
+
AUTODATA_OLLAMA_NUM_PREDICT=1200
|
| 9 |
+
AUTODATA_LLM_TIMEOUT_SECONDS=90
|
| 10 |
+
AUTODATA_EXECUTION_TIMEOUT_SECONDS=12
|
| 11 |
+
AUTODATA_MAX_REPAIR_ATTEMPTS=2
|
| 12 |
+
AUTODATA_CORS_ORIGINS=http://localhost:5173,http://localhost:8501,http://127.0.0.1:5173,http://127.0.0.1:8501
|
Dockerfile
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
|
| 3 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 4 |
+
PIP_NO_CACHE_DIR=1 \
|
| 5 |
+
AUTODATA_ENV=production \
|
| 6 |
+
AUTODATA_STORAGE_DIR=/tmp/autodata/runtime \
|
| 7 |
+
AUTODATA_UPLOAD_DIR=/tmp/autodata/uploads
|
| 8 |
+
|
| 9 |
+
WORKDIR /app
|
| 10 |
+
|
| 11 |
+
RUN apt-get update && \
|
| 12 |
+
apt-get install -y --no-install-recommends build-essential curl && \
|
| 13 |
+
rm -rf /var/lib/apt/lists/*
|
| 14 |
+
|
| 15 |
+
COPY pyproject.toml README.md ./
|
| 16 |
+
COPY src ./src
|
| 17 |
+
|
| 18 |
+
RUN pip install --upgrade pip && \
|
| 19 |
+
pip install .
|
| 20 |
+
|
| 21 |
+
EXPOSE 7860
|
| 22 |
+
|
| 23 |
+
CMD ["uvicorn", "autodata_agent.api.app:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
|
@@ -1,10 +1,177 @@
|
|
| 1 |
-
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
-
sdk: docker
|
| 7 |
-
|
| 8 |
-
---
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: ADAA Agent Backend
|
| 3 |
+
emoji: 📊
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
# Autonomous Data Analysis Agent Backend
|
| 11 |
+
|
| 12 |
+
Python backend for Assignment 9: Autonomous Data Analysis Agent.
|
| 13 |
+
|
| 14 |
+
The backend accepts structured data, profiles it, sends business questions to an Ollama cloud model, executes generated analysis code in a controlled runner, repairs failed generated code, returns chart metadata and a business narrative, and stores session history for follow-up questions.
|
| 15 |
+
|
| 16 |
+
## Current Default Model
|
| 17 |
+
|
| 18 |
+
Default for direct Ollama Cloud API in this repo: `qwen3-coder:480b`
|
| 19 |
+
|
| 20 |
+
Reason: `qwen3-coder:480b` is a cloud coding model available to the current API key and is suited for generated Python/pandas analysis code. The best premium choice remains `kimi-k2.7-code`, but the current account receives a subscription-required error for that model. `glm-5.1` is also a strong agentic engineering model, but it currently returns the same subscription-required error on this key.
|
| 21 |
+
|
| 22 |
+
References:
|
| 23 |
+
- https://ollama.com/library/kimi-k2.7-code
|
| 24 |
+
- https://ollama.com/library/glm-5.1
|
| 25 |
+
- https://docs.ollama.com/api/openai-compatibility
|
| 26 |
+
|
| 27 |
+
## Features Implemented
|
| 28 |
+
|
| 29 |
+
- CSV, Excel, JSON ingestion.
|
| 30 |
+
- SQL ingestion through SQLAlchemy for SELECT queries.
|
| 31 |
+
- Automatic dataset profiling: schema, data types, null rates, distributions, duplicate rows, and anomaly hints.
|
| 32 |
+
- Ollama Cloud client using native `/api/chat`.
|
| 33 |
+
- Generated Python analysis code contract: `result_df` and `chart_spec`.
|
| 34 |
+
- Controlled subprocess execution with blocked unsafe operations and timeout.
|
| 35 |
+
- Error repair loop with configurable retry count.
|
| 36 |
+
- Chart specification validation for bar, line, scatter, heatmap, distribution, and table outputs.
|
| 37 |
+
- Findings narrative generation with limitations and follow-up questions.
|
| 38 |
+
- SQLite-backed session history.
|
| 39 |
+
- Demo Excel workbook, CSV, and JSON fixtures under `data/`.
|
| 40 |
+
|
| 41 |
+
## Setup
|
| 42 |
+
|
| 43 |
+
```powershell
|
| 44 |
+
python -m venv .venv
|
| 45 |
+
.venv\Scripts\Activate.ps1
|
| 46 |
+
python -m pip install --upgrade pip
|
| 47 |
+
python -m pip install -e .[dev]
|
| 48 |
+
Copy-Item .env.example .env
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
Edit `.env`:
|
| 52 |
+
|
| 53 |
+
```text
|
| 54 |
+
AUTODATA_OLLAMA_API_KEY=your_ollama_key
|
| 55 |
+
AUTODATA_OLLAMA_BASE_URL=https://ollama.com
|
| 56 |
+
AUTODATA_OLLAMA_MODEL=qwen3-coder:480b
|
| 57 |
+
AUTODATA_OLLAMA_THINK=false
|
| 58 |
+
AUTODATA_OLLAMA_NUM_PREDICT=1200
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Run the API:
|
| 62 |
+
|
| 63 |
+
```powershell
|
| 64 |
+
uvicorn autodata_agent.api.app:app --reload
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
Open:
|
| 68 |
+
|
| 69 |
+
```text
|
| 70 |
+
http://127.0.0.1:8000/docs
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
Check Ollama Cloud connectivity:
|
| 74 |
+
|
| 75 |
+
```powershell
|
| 76 |
+
python scripts/check_ollama_connection.py
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
## Main API Endpoints
|
| 80 |
+
|
| 81 |
+
| Method | Path | Purpose |
|
| 82 |
+
|---|---|---|
|
| 83 |
+
| GET | `/api/v1/health` | Backend health and configured model |
|
| 84 |
+
| POST | `/api/v1/datasets/upload` | Upload CSV/XLSX/JSON |
|
| 85 |
+
| POST | `/api/v1/datasets/sql` | Ingest SQL SELECT query |
|
| 86 |
+
| GET | `/api/v1/datasets/{dataset_id}/profile` | View automatic profile |
|
| 87 |
+
| GET | `/api/v1/datasets/{dataset_id}/preview` | Preview uploaded dataset rows |
|
| 88 |
+
| POST | `/api/v1/analysis` | Ask a business question |
|
| 89 |
+
| GET | `/api/v1/sessions/{session_id}/history` | Read session history |
|
| 90 |
+
| GET | `/api/v1/sessions/{session_id}/export` | Download a JSON report of all analyses in a session |
|
| 91 |
+
|
| 92 |
+
### Session Export
|
| 93 |
+
|
| 94 |
+
`GET /api/v1/sessions/{session_id}/export` returns a clean JSON report of every analysis run in a session, in chronological order. It reads only stored session records — it does not call Ollama or rerun generated code. An empty or unknown session returns an empty `analyses` list.
|
| 95 |
+
|
| 96 |
+
```json
|
| 97 |
+
{
|
| 98 |
+
"session_id": "...",
|
| 99 |
+
"exported_at": "2026-06-18T...",
|
| 100 |
+
"analyses": [
|
| 101 |
+
{
|
| 102 |
+
"question": "Which product categories generated the highest total revenue?",
|
| 103 |
+
"operation": "aggregation",
|
| 104 |
+
"generated_code": "...",
|
| 105 |
+
"result_rows": [{ "category": "A", "sales": 150 }],
|
| 106 |
+
"chart_spec": { "chart_type": "bar", "title": "Sales by Category", "...": "..." },
|
| 107 |
+
"narrative": {
|
| 108 |
+
"key_finding": "...",
|
| 109 |
+
"business_meaning": "...",
|
| 110 |
+
"limitations": ["..."],
|
| 111 |
+
"follow_up_questions": ["..."]
|
| 112 |
+
}
|
| 113 |
+
}
|
| 114 |
+
]
|
| 115 |
+
}
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
## Example Request Flow
|
| 119 |
+
|
| 120 |
+
1. Upload `data/autodata_demo_business_dataset.xlsx` to `/api/v1/datasets/upload`.
|
| 121 |
+
2. Copy the returned `dataset_id`.
|
| 122 |
+
3. POST to `/api/v1/analysis`:
|
| 123 |
+
|
| 124 |
+
```json
|
| 125 |
+
{
|
| 126 |
+
"dataset_id": "paste_dataset_id_here",
|
| 127 |
+
"question": "Which product categories generated the highest total revenue?"
|
| 128 |
+
}
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
4. Reuse the returned `session_id` for follow-up questions:
|
| 132 |
+
|
| 133 |
+
```json
|
| 134 |
+
{
|
| 135 |
+
"dataset_id": "paste_dataset_id_here",
|
| 136 |
+
"session_id": "paste_session_id_here",
|
| 137 |
+
"question": "For the weakest-margin segment, which categories are causing the problem?"
|
| 138 |
+
}
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
## Testing
|
| 142 |
+
|
| 143 |
+
```powershell
|
| 144 |
+
pytest
|
| 145 |
+
ruff check .
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
The tests use fake LLM clients where needed. Production analysis does not silently fall back when Ollama is missing; it returns a structured service error.
|
| 149 |
+
|
| 150 |
+
## Publishing With GitHub Desktop
|
| 151 |
+
|
| 152 |
+
This workspace is already initialized as a git repository on `main` with logical commits.
|
| 153 |
+
|
| 154 |
+
To publish the remote repository:
|
| 155 |
+
|
| 156 |
+
1. Open GitHub Desktop.
|
| 157 |
+
2. Add/open this local repository:
|
| 158 |
+
`C:\Users\lostdecimal27\Documents\Codex\2026-06-15\gen-ai-project-agent`
|
| 159 |
+
3. Click **Publish repository**.
|
| 160 |
+
4. Recommended repository name: `autonomous-data-analysis-agent-backend`.
|
| 161 |
+
5. Keep the repo private until submission details are final, unless your course requires public access.
|
| 162 |
+
|
| 163 |
+
## Demo Data
|
| 164 |
+
|
| 165 |
+
Generated files:
|
| 166 |
+
|
| 167 |
+
- `data/autodata_demo_business_dataset.xlsx`
|
| 168 |
+
- `data/autodata_demo_orders.csv`
|
| 169 |
+
- `data/autodata_demo_orders.json`
|
| 170 |
+
|
| 171 |
+
Regenerate them with:
|
| 172 |
+
|
| 173 |
+
```powershell
|
| 174 |
+
node scripts/build_demo_workbook.mjs
|
| 175 |
+
```
|
| 176 |
+
|
| 177 |
+
The workbook includes a dashboard, order-level data, summary tables, a data dictionary, demo questions, and QA checks.
|
pyproject.toml
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=69", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "autodata-agent-backend"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "Backend for an autonomous data analysis agent."
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.11"
|
| 11 |
+
dependencies = [
|
| 12 |
+
"fastapi>=0.115",
|
| 13 |
+
"uvicorn[standard]>=0.30",
|
| 14 |
+
"pydantic>=2.8",
|
| 15 |
+
"pydantic-settings>=2.3",
|
| 16 |
+
"python-multipart>=0.0.9",
|
| 17 |
+
"pandas>=2.2",
|
| 18 |
+
"numpy>=1.26",
|
| 19 |
+
"openpyxl>=3.1",
|
| 20 |
+
"duckdb>=1.0",
|
| 21 |
+
"sqlalchemy>=2.0",
|
| 22 |
+
"httpx>=0.27",
|
| 23 |
+
"plotly>=5.22",
|
| 24 |
+
]
|
| 25 |
+
|
| 26 |
+
[project.optional-dependencies]
|
| 27 |
+
dev = [
|
| 28 |
+
"pytest>=8.2",
|
| 29 |
+
"pytest-cov>=5.0",
|
| 30 |
+
"ruff>=0.5",
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
[tool.setuptools.packages.find]
|
| 34 |
+
where = ["src"]
|
| 35 |
+
|
| 36 |
+
[tool.pytest.ini_options]
|
| 37 |
+
testpaths = ["tests"]
|
| 38 |
+
addopts = "-q"
|
| 39 |
+
|
| 40 |
+
[tool.ruff]
|
| 41 |
+
line-length = 100
|
| 42 |
+
target-version = "py311"
|
| 43 |
+
|
| 44 |
+
[tool.ruff.lint]
|
| 45 |
+
select = ["E", "F", "I", "UP", "B"]
|
| 46 |
+
|
src/autodata_agent/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Autonomous Data Analysis Agent backend."""
|
| 2 |
+
|
| 3 |
+
__all__ = ["__version__"]
|
| 4 |
+
|
| 5 |
+
__version__ = "0.1.0"
|
| 6 |
+
|
src/autodata_agent/api/app.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from fastapi import FastAPI, Request
|
| 4 |
+
from fastapi.exceptions import RequestValidationError
|
| 5 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 6 |
+
from fastapi.responses import JSONResponse
|
| 7 |
+
|
| 8 |
+
from autodata_agent.api.routes import router
|
| 9 |
+
from autodata_agent.core.config import get_settings
|
| 10 |
+
from autodata_agent.core.errors import AppError
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _error_payload(code: str, message: str, details: dict | list | None = None) -> dict:
|
| 14 |
+
return {"error": {"code": code, "message": message, "details": details or {}}}
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def create_app() -> FastAPI:
|
| 18 |
+
settings = get_settings()
|
| 19 |
+
app = FastAPI(
|
| 20 |
+
title="Autonomous Data Analysis Agent Backend",
|
| 21 |
+
version="0.1.0",
|
| 22 |
+
description=(
|
| 23 |
+
"Backend API for data ingestion, autonomous analysis, visualization specs, "
|
| 24 |
+
"and history."
|
| 25 |
+
),
|
| 26 |
+
)
|
| 27 |
+
if settings.cors_origins:
|
| 28 |
+
app.add_middleware(
|
| 29 |
+
CORSMiddleware,
|
| 30 |
+
allow_origins=settings.cors_origins,
|
| 31 |
+
allow_credentials=True,
|
| 32 |
+
allow_methods=["*"],
|
| 33 |
+
allow_headers=["*"],
|
| 34 |
+
)
|
| 35 |
+
app.include_router(router, prefix="/api/v1")
|
| 36 |
+
|
| 37 |
+
@app.exception_handler(AppError)
|
| 38 |
+
async def app_error_handler(_: Request, exc: AppError) -> JSONResponse:
|
| 39 |
+
return JSONResponse(
|
| 40 |
+
status_code=exc.status_code,
|
| 41 |
+
content=_error_payload(exc.code, exc.message, exc.details),
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
@app.exception_handler(RequestValidationError)
|
| 45 |
+
async def validation_error_handler(_: Request, exc: RequestValidationError) -> JSONResponse:
|
| 46 |
+
return JSONResponse(
|
| 47 |
+
status_code=422,
|
| 48 |
+
content=_error_payload(
|
| 49 |
+
"request_validation_failed",
|
| 50 |
+
"Request validation failed.",
|
| 51 |
+
exc.errors(),
|
| 52 |
+
),
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
@app.exception_handler(Exception)
|
| 56 |
+
async def unhandled_error_handler(_: Request, exc: Exception) -> JSONResponse:
|
| 57 |
+
return JSONResponse(
|
| 58 |
+
status_code=500,
|
| 59 |
+
content={
|
| 60 |
+
"error": {
|
| 61 |
+
"code": "internal_server_error",
|
| 62 |
+
"message": "An unexpected backend error occurred.",
|
| 63 |
+
"details": {"reason": str(exc)},
|
| 64 |
+
}
|
| 65 |
+
},
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
return app
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
app = create_app()
|
src/autodata_agent/api/dependencies.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from functools import lru_cache
|
| 4 |
+
|
| 5 |
+
from autodata_agent.core.config import Settings, get_settings
|
| 6 |
+
from autodata_agent.services.analysis import AnalysisService
|
| 7 |
+
from autodata_agent.services.datasets import DatasetStore
|
| 8 |
+
from autodata_agent.services.executor import CodeExecutor
|
| 9 |
+
from autodata_agent.services.llm import OllamaCloudClient
|
| 10 |
+
from autodata_agent.storage.session_store import SessionStore
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@lru_cache
|
| 14 |
+
def get_dataset_store() -> DatasetStore:
|
| 15 |
+
settings = get_settings()
|
| 16 |
+
return DatasetStore(
|
| 17 |
+
settings.upload_dir,
|
| 18 |
+
settings.max_upload_bytes,
|
| 19 |
+
settings.storage_dir / "datasets",
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@lru_cache
|
| 24 |
+
def get_session_store() -> SessionStore:
|
| 25 |
+
settings = get_settings()
|
| 26 |
+
return SessionStore(settings.storage_dir / "sessions.sqlite3")
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def get_analysis_service() -> AnalysisService:
|
| 30 |
+
settings = get_settings()
|
| 31 |
+
return AnalysisService(
|
| 32 |
+
datasets=get_dataset_store(),
|
| 33 |
+
sessions=get_session_store(),
|
| 34 |
+
llm=OllamaCloudClient(settings),
|
| 35 |
+
executor=CodeExecutor(settings.execution_timeout_seconds),
|
| 36 |
+
max_repair_attempts=settings.max_repair_attempts,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def settings_dependency() -> Settings:
|
| 41 |
+
return get_settings()
|
src/autodata_agent/api/routes.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Annotated
|
| 4 |
+
|
| 5 |
+
from fastapi import APIRouter, Depends, File, Query, UploadFile
|
| 6 |
+
|
| 7 |
+
from autodata_agent.api.dependencies import (
|
| 8 |
+
get_analysis_service,
|
| 9 |
+
get_dataset_store,
|
| 10 |
+
get_session_store,
|
| 11 |
+
settings_dependency,
|
| 12 |
+
)
|
| 13 |
+
from autodata_agent.core.config import Settings
|
| 14 |
+
from autodata_agent.core.schemas import (
|
| 15 |
+
AnalysisRequest,
|
| 16 |
+
AnalysisResponse,
|
| 17 |
+
ChartRenderRequest,
|
| 18 |
+
ChartRenderResponse,
|
| 19 |
+
DatasetInfo,
|
| 20 |
+
DatasetPreview,
|
| 21 |
+
DatasetProfile,
|
| 22 |
+
HealthStatus,
|
| 23 |
+
SessionExport,
|
| 24 |
+
SessionRecord,
|
| 25 |
+
SQLIngestRequest,
|
| 26 |
+
)
|
| 27 |
+
from autodata_agent.services.analysis import AnalysisService
|
| 28 |
+
from autodata_agent.services.charts import render_chart_figure
|
| 29 |
+
from autodata_agent.services.datasets import DatasetStore
|
| 30 |
+
from autodata_agent.storage.session_store import SessionStore
|
| 31 |
+
|
| 32 |
+
router = APIRouter()
|
| 33 |
+
|
| 34 |
+
SettingsDep = Annotated[Settings, Depends(settings_dependency)]
|
| 35 |
+
DatasetStoreDep = Annotated[DatasetStore, Depends(get_dataset_store)]
|
| 36 |
+
SessionStoreDep = Annotated[SessionStore, Depends(get_session_store)]
|
| 37 |
+
AnalysisServiceDep = Annotated[AnalysisService, Depends(get_analysis_service)]
|
| 38 |
+
UploadFileDep = Annotated[UploadFile, File(...)]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@router.get("/health", response_model=HealthStatus)
|
| 42 |
+
def health(settings: SettingsDep) -> HealthStatus:
|
| 43 |
+
return HealthStatus(status="ok", environment=settings.env, model=settings.ollama_model)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@router.post("/datasets/upload", response_model=DatasetInfo)
|
| 47 |
+
async def upload_dataset(
|
| 48 |
+
file: UploadFileDep,
|
| 49 |
+
datasets: DatasetStoreDep,
|
| 50 |
+
) -> DatasetInfo:
|
| 51 |
+
content = await file.read()
|
| 52 |
+
return datasets.put_file(file.filename or "uploaded_dataset", content)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@router.post("/datasets/sql", response_model=DatasetInfo)
|
| 56 |
+
def ingest_sql(
|
| 57 |
+
request: SQLIngestRequest,
|
| 58 |
+
datasets: DatasetStoreDep,
|
| 59 |
+
) -> DatasetInfo:
|
| 60 |
+
return datasets.put_sql(request)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
@router.get("/datasets/{dataset_id}/profile", response_model=DatasetProfile)
|
| 64 |
+
def get_profile(
|
| 65 |
+
dataset_id: str,
|
| 66 |
+
datasets: DatasetStoreDep,
|
| 67 |
+
) -> DatasetProfile:
|
| 68 |
+
return datasets.get(dataset_id).profile
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@router.get("/datasets/{dataset_id}/preview", response_model=DatasetPreview)
|
| 72 |
+
def get_preview(
|
| 73 |
+
dataset_id: str,
|
| 74 |
+
datasets: DatasetStoreDep,
|
| 75 |
+
limit: int = Query(20, ge=1, le=1000),
|
| 76 |
+
) -> DatasetPreview:
|
| 77 |
+
return datasets.preview(dataset_id, limit)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@router.post("/analysis", response_model=AnalysisResponse)
|
| 81 |
+
def analyze(
|
| 82 |
+
request: AnalysisRequest,
|
| 83 |
+
service: AnalysisServiceDep,
|
| 84 |
+
) -> AnalysisResponse:
|
| 85 |
+
return service.analyze(request)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@router.post("/charts/render", response_model=ChartRenderResponse)
|
| 89 |
+
def render_chart(request: ChartRenderRequest) -> ChartRenderResponse:
|
| 90 |
+
figure = render_chart_figure(request.chart_spec, request.result_rows)
|
| 91 |
+
return ChartRenderResponse(
|
| 92 |
+
chart_spec=request.chart_spec,
|
| 93 |
+
figure=figure,
|
| 94 |
+
caption=request.chart_spec.caption,
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@router.get("/sessions/{session_id}/history", response_model=list[SessionRecord])
|
| 99 |
+
def session_history(
|
| 100 |
+
session_id: str,
|
| 101 |
+
sessions: SessionStoreDep,
|
| 102 |
+
) -> list[SessionRecord]:
|
| 103 |
+
return sessions.history(session_id)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
@router.get("/sessions/{session_id}/export", response_model=SessionExport)
|
| 107 |
+
def export_session(
|
| 108 |
+
session_id: str,
|
| 109 |
+
sessions: SessionStoreDep,
|
| 110 |
+
) -> SessionExport:
|
| 111 |
+
return SessionExport.from_records(session_id, sessions.all_records(session_id))
|
src/autodata_agent/core/config.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from functools import lru_cache
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
from pydantic import Field, SecretStr, field_validator
|
| 7 |
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
| 8 |
+
|
| 9 |
+
DEFAULT_CORS_ORIGINS = (
|
| 10 |
+
"http://localhost:5173",
|
| 11 |
+
"http://localhost:8501",
|
| 12 |
+
"http://127.0.0.1:5173",
|
| 13 |
+
"http://127.0.0.1:8501",
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class Settings(BaseSettings):
|
| 18 |
+
model_config = SettingsConfigDict(
|
| 19 |
+
env_prefix="AUTODATA_",
|
| 20 |
+
env_file=".env",
|
| 21 |
+
env_file_encoding="utf-8",
|
| 22 |
+
extra="ignore",
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
env: str = "development"
|
| 26 |
+
storage_dir: Path = Path("data/runtime")
|
| 27 |
+
upload_dir: Path = Path("data/uploads")
|
| 28 |
+
|
| 29 |
+
ollama_base_url: str = "https://ollama.com"
|
| 30 |
+
ollama_api_key: SecretStr | None = None
|
| 31 |
+
ollama_model: str = "qwen3-coder:480b"
|
| 32 |
+
ollama_think: str = "false"
|
| 33 |
+
ollama_num_predict: int = 1200
|
| 34 |
+
|
| 35 |
+
llm_timeout_seconds: float = 90
|
| 36 |
+
execution_timeout_seconds: float = 12
|
| 37 |
+
max_repair_attempts: int = Field(default=2, ge=0, le=5)
|
| 38 |
+
max_upload_bytes: int = 50 * 1024 * 1024
|
| 39 |
+
max_preview_rows: int = 50
|
| 40 |
+
cors_origins: list[str] = Field(default_factory=lambda: list(DEFAULT_CORS_ORIGINS))
|
| 41 |
+
|
| 42 |
+
@field_validator("cors_origins", mode="before")
|
| 43 |
+
@classmethod
|
| 44 |
+
def parse_cors_origins(cls, value: object) -> list[str]:
|
| 45 |
+
if value is None or value == "":
|
| 46 |
+
return []
|
| 47 |
+
if isinstance(value, str):
|
| 48 |
+
return [origin.strip() for origin in value.split(",") if origin.strip()]
|
| 49 |
+
return value # type: ignore[return-value]
|
| 50 |
+
|
| 51 |
+
def ensure_dirs(self) -> None:
|
| 52 |
+
self.storage_dir.mkdir(parents=True, exist_ok=True)
|
| 53 |
+
self.upload_dir.mkdir(parents=True, exist_ok=True)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@lru_cache
|
| 57 |
+
def get_settings() -> Settings:
|
| 58 |
+
settings = Settings()
|
| 59 |
+
settings.ensure_dirs()
|
| 60 |
+
return settings
|
src/autodata_agent/core/errors.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class AppError(Exception):
|
| 7 |
+
"""Structured application error that can be safely returned through the API."""
|
| 8 |
+
|
| 9 |
+
def __init__(
|
| 10 |
+
self,
|
| 11 |
+
code: str,
|
| 12 |
+
message: str,
|
| 13 |
+
*,
|
| 14 |
+
status_code: int = 400,
|
| 15 |
+
details: dict[str, Any] | None = None,
|
| 16 |
+
) -> None:
|
| 17 |
+
super().__init__(message)
|
| 18 |
+
self.code = code
|
| 19 |
+
self.message = message
|
| 20 |
+
self.status_code = status_code
|
| 21 |
+
self.details = details or {}
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class ExternalServiceError(AppError):
|
| 25 |
+
pass
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class ValidationAppError(AppError):
|
| 29 |
+
pass
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class ExecutionAppError(AppError):
|
| 33 |
+
pass
|
| 34 |
+
|
src/autodata_agent/core/json_utils.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import re
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from autodata_agent.core.errors import ValidationAppError
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def extract_json_object(text: str) -> dict[str, Any]:
|
| 11 |
+
"""Extract one JSON object from an LLM response."""
|
| 12 |
+
|
| 13 |
+
stripped = text.strip()
|
| 14 |
+
if stripped.startswith("```"):
|
| 15 |
+
stripped = re.sub(r"^```(?:json)?", "", stripped, flags=re.IGNORECASE).strip()
|
| 16 |
+
stripped = re.sub(r"```$", "", stripped).strip()
|
| 17 |
+
|
| 18 |
+
try:
|
| 19 |
+
value = json.loads(stripped)
|
| 20 |
+
except json.JSONDecodeError:
|
| 21 |
+
start = stripped.find("{")
|
| 22 |
+
end = stripped.rfind("}")
|
| 23 |
+
if start == -1 or end == -1 or end <= start:
|
| 24 |
+
raise ValidationAppError(
|
| 25 |
+
"invalid_llm_json",
|
| 26 |
+
"The model did not return a JSON object.",
|
| 27 |
+
details={"response_preview": stripped[:500]},
|
| 28 |
+
) from None
|
| 29 |
+
try:
|
| 30 |
+
value = json.loads(stripped[start : end + 1])
|
| 31 |
+
except json.JSONDecodeError as exc:
|
| 32 |
+
raise ValidationAppError(
|
| 33 |
+
"invalid_llm_json",
|
| 34 |
+
"The model returned malformed JSON.",
|
| 35 |
+
details={"response_preview": stripped[:500], "json_error": str(exc)},
|
| 36 |
+
) from exc
|
| 37 |
+
|
| 38 |
+
if not isinstance(value, dict):
|
| 39 |
+
raise ValidationAppError("invalid_llm_json", "The model response JSON must be an object.")
|
| 40 |
+
return value
|
| 41 |
+
|
src/autodata_agent/core/schemas.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from datetime import UTC, datetime
|
| 4 |
+
from enum import StrEnum
|
| 5 |
+
from typing import Any, Literal
|
| 6 |
+
|
| 7 |
+
from pydantic import BaseModel, Field, field_validator
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class ChartType(StrEnum):
|
| 11 |
+
BAR = "bar"
|
| 12 |
+
LINE = "line"
|
| 13 |
+
SCATTER = "scatter"
|
| 14 |
+
HEATMAP = "heatmap"
|
| 15 |
+
DISTRIBUTION = "distribution"
|
| 16 |
+
TABLE = "table"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class DataSourceType(StrEnum):
|
| 20 |
+
CSV = "csv"
|
| 21 |
+
EXCEL = "excel"
|
| 22 |
+
JSON = "json"
|
| 23 |
+
SQL = "sql"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class ColumnProfile(BaseModel):
|
| 27 |
+
name: str
|
| 28 |
+
dtype: str
|
| 29 |
+
non_null_count: int
|
| 30 |
+
null_count: int
|
| 31 |
+
null_rate: float
|
| 32 |
+
unique_count: int
|
| 33 |
+
sample_values: list[Any] = Field(default_factory=list)
|
| 34 |
+
summary: dict[str, Any] = Field(default_factory=dict)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class DatasetProfile(BaseModel):
|
| 38 |
+
dataset_id: str
|
| 39 |
+
source_name: str
|
| 40 |
+
source_type: DataSourceType
|
| 41 |
+
row_count: int
|
| 42 |
+
column_count: int
|
| 43 |
+
duplicate_rows: int
|
| 44 |
+
columns: list[ColumnProfile]
|
| 45 |
+
anomalies: list[str] = Field(default_factory=list)
|
| 46 |
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class DatasetInfo(BaseModel):
|
| 50 |
+
dataset_id: str
|
| 51 |
+
source_name: str
|
| 52 |
+
source_type: DataSourceType
|
| 53 |
+
row_count: int
|
| 54 |
+
column_count: int
|
| 55 |
+
profile: DatasetProfile
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class SQLIngestRequest(BaseModel):
|
| 59 |
+
connection_uri: str = Field(..., description="SQLAlchemy connection URI.")
|
| 60 |
+
query: str = Field(..., description="SELECT query to load into the analysis dataset.")
|
| 61 |
+
source_name: str = "sql_dataset"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class AnalysisOperation(StrEnum):
|
| 65 |
+
OVERVIEW = "overview"
|
| 66 |
+
AGGREGATION = "aggregation"
|
| 67 |
+
FILTERING = "filtering"
|
| 68 |
+
GROUPING = "grouping"
|
| 69 |
+
TREND = "trend_analysis"
|
| 70 |
+
CORRELATION = "correlation"
|
| 71 |
+
SEGMENTATION = "segmentation"
|
| 72 |
+
DISTRIBUTION = "distribution"
|
| 73 |
+
ANOMALY = "anomaly_detection"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class AnalysisPlan(BaseModel):
|
| 77 |
+
operation: AnalysisOperation
|
| 78 |
+
objective: str
|
| 79 |
+
required_columns: list[str]
|
| 80 |
+
assumptions: list[str] = Field(default_factory=list)
|
| 81 |
+
chart_type: ChartType
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class ChartSpec(BaseModel):
|
| 85 |
+
chart_type: ChartType
|
| 86 |
+
title: str
|
| 87 |
+
x: str | None = None
|
| 88 |
+
y: str | None = None
|
| 89 |
+
color: str | None = None
|
| 90 |
+
caption: str
|
| 91 |
+
|
| 92 |
+
@field_validator("x", "y", "color", mode="before")
|
| 93 |
+
@classmethod
|
| 94 |
+
def normalize_optional_axis(cls, value: Any) -> Any:
|
| 95 |
+
if value == [] or value == "":
|
| 96 |
+
return None
|
| 97 |
+
if isinstance(value, list):
|
| 98 |
+
return str(value[0]) if value else None
|
| 99 |
+
return value
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class Narrative(BaseModel):
|
| 103 |
+
key_finding: str
|
| 104 |
+
business_meaning: str
|
| 105 |
+
limitations: list[str]
|
| 106 |
+
follow_up_questions: list[str]
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
class GeneratedAnalysisCode(BaseModel):
|
| 110 |
+
plan: AnalysisPlan
|
| 111 |
+
code: str = Field(..., description="Python code that creates result_df and chart_spec.")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class ExecutionResult(BaseModel):
|
| 115 |
+
success: bool
|
| 116 |
+
result_columns: list[str] = Field(default_factory=list)
|
| 117 |
+
result_rows: list[dict[str, Any]] = Field(default_factory=list)
|
| 118 |
+
chart_spec: ChartSpec | None = None
|
| 119 |
+
stdout: str = ""
|
| 120 |
+
error: str | None = None
|
| 121 |
+
attempts: int = 1
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
class AnalysisRequest(BaseModel):
|
| 125 |
+
dataset_id: str
|
| 126 |
+
question: str
|
| 127 |
+
session_id: str | None = None
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
class AnalysisResponse(BaseModel):
|
| 131 |
+
session_id: str
|
| 132 |
+
dataset_id: str
|
| 133 |
+
question: str
|
| 134 |
+
plan: AnalysisPlan
|
| 135 |
+
generated_code: str
|
| 136 |
+
execution: ExecutionResult
|
| 137 |
+
narrative: Narrative
|
| 138 |
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
class SessionRecord(BaseModel):
|
| 142 |
+
record_id: str
|
| 143 |
+
session_id: str
|
| 144 |
+
dataset_id: str
|
| 145 |
+
question: str
|
| 146 |
+
response: AnalysisResponse
|
| 147 |
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class AnalysisExport(BaseModel):
|
| 151 |
+
question: str
|
| 152 |
+
operation: AnalysisOperation
|
| 153 |
+
generated_code: str
|
| 154 |
+
result_rows: list[dict[str, Any]] = Field(default_factory=list)
|
| 155 |
+
chart_spec: ChartSpec | None = None
|
| 156 |
+
narrative: Narrative
|
| 157 |
+
|
| 158 |
+
@classmethod
|
| 159 |
+
def from_record(cls, record: SessionRecord) -> AnalysisExport:
|
| 160 |
+
response = record.response
|
| 161 |
+
return cls(
|
| 162 |
+
question=record.question,
|
| 163 |
+
operation=response.plan.operation,
|
| 164 |
+
generated_code=response.generated_code,
|
| 165 |
+
result_rows=response.execution.result_rows,
|
| 166 |
+
chart_spec=response.execution.chart_spec,
|
| 167 |
+
narrative=response.narrative,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
class SessionExport(BaseModel):
|
| 172 |
+
session_id: str
|
| 173 |
+
exported_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
| 174 |
+
analyses: list[AnalysisExport] = Field(default_factory=list)
|
| 175 |
+
|
| 176 |
+
@classmethod
|
| 177 |
+
def from_records(cls, session_id: str, records: list[SessionRecord]) -> SessionExport:
|
| 178 |
+
return cls(
|
| 179 |
+
session_id=session_id,
|
| 180 |
+
analyses=[AnalysisExport.from_record(record) for record in records],
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class HealthStatus(BaseModel):
|
| 185 |
+
status: Literal["ok"]
|
| 186 |
+
environment: str
|
| 187 |
+
model: str
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
class ErrorResponse(BaseModel):
|
| 191 |
+
error: dict[str, Any]
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
class DatasetPreview(BaseModel):
|
| 195 |
+
dataset_id: str
|
| 196 |
+
columns: list[str]
|
| 197 |
+
rows: list[dict[str, Any]]
|
| 198 |
+
row_count: int
|
| 199 |
+
returned_rows: int
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class ChartRenderRequest(BaseModel):
|
| 203 |
+
chart_spec: ChartSpec
|
| 204 |
+
result_rows: list[dict[str, Any]] = Field(default_factory=list)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
class ChartRenderResponse(BaseModel):
|
| 208 |
+
chart_spec: ChartSpec
|
| 209 |
+
figure: dict[str, Any]
|
| 210 |
+
caption: str
|
| 211 |
+
|
src/autodata_agent/main.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import uvicorn
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def main() -> None:
|
| 7 |
+
uvicorn.run("autodata_agent.api.app:app", host="0.0.0.0", port=8000, reload=True)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
if __name__ == "__main__":
|
| 11 |
+
main()
|
| 12 |
+
|
src/autodata_agent/services/analysis.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
from autodata_agent.core.errors import ExecutionAppError, ValidationAppError
|
| 6 |
+
from autodata_agent.core.json_utils import extract_json_object
|
| 7 |
+
from autodata_agent.core.schemas import (
|
| 8 |
+
AnalysisOperation,
|
| 9 |
+
AnalysisPlan,
|
| 10 |
+
AnalysisRequest,
|
| 11 |
+
AnalysisResponse,
|
| 12 |
+
ChartType,
|
| 13 |
+
ExecutionResult,
|
| 14 |
+
GeneratedAnalysisCode,
|
| 15 |
+
Narrative,
|
| 16 |
+
)
|
| 17 |
+
from autodata_agent.services.datasets import DatasetStore
|
| 18 |
+
from autodata_agent.services.executor import CodeExecutor, compact_result_for_prompt
|
| 19 |
+
from autodata_agent.services.llm import LLMClient
|
| 20 |
+
from autodata_agent.storage.session_store import SessionStore
|
| 21 |
+
|
| 22 |
+
GENERATION_SYSTEM_PROMPT = """You are the code-generation brain for a data analysis backend.
|
| 23 |
+
Return only valid JSON matching this exact schema:
|
| 24 |
+
{
|
| 25 |
+
"plan": {
|
| 26 |
+
"operation": "one allowed operation value",
|
| 27 |
+
"objective": "short objective",
|
| 28 |
+
"required_columns": ["only columns that exist in the dataset"],
|
| 29 |
+
"assumptions": ["short assumptions"],
|
| 30 |
+
"chart_type": "bar|line|scatter|heatmap|distribution|table"
|
| 31 |
+
},
|
| 32 |
+
"code": "Python code as one string. It must define result_df and chart_spec."
|
| 33 |
+
}
|
| 34 |
+
Allowed operation values: overview, aggregation, filtering, grouping, trend_analysis,
|
| 35 |
+
correlation, segmentation, distribution, anomaly_detection.
|
| 36 |
+
Do not return top-level result_df. Do not return top-level chart_spec. Do not include markdown.
|
| 37 |
+
Generate Python/pandas code against an existing DataFrame named df.
|
| 38 |
+
The code must define:
|
| 39 |
+
1. result_df: a pandas DataFrame containing the final answer.
|
| 40 |
+
2. chart_spec: a dict with chart_type, title, x, y, optional color, and caption.
|
| 41 |
+
Do not import modules. Use only pd, np, duckdb, math, statistics, and df.
|
| 42 |
+
Do not read files, call networks, access environment variables, or mutate external state.
|
| 43 |
+
Choose chart_type from: bar, line, scatter, heatmap, distribution, table.
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
REPAIR_SYSTEM_PROMPT = """You repair failed pandas analysis code.
|
| 48 |
+
Return only valid JSON with the same schema as the original generation.
|
| 49 |
+
Do not include markdown. Do not import modules. Keep the output contract unchanged.
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
NARRATIVE_SYSTEM_PROMPT = """You write concise business findings for data analysis results.
|
| 54 |
+
Return only valid JSON with key_finding, business_meaning, limitations, follow_up_questions.
|
| 55 |
+
State uncertainty and limitations clearly. Do not invent facts not supported by result rows.
|
| 56 |
+
"""
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class AnalysisService:
|
| 60 |
+
def __init__(
|
| 61 |
+
self,
|
| 62 |
+
*,
|
| 63 |
+
datasets: DatasetStore,
|
| 64 |
+
sessions: SessionStore,
|
| 65 |
+
llm: LLMClient,
|
| 66 |
+
executor: CodeExecutor,
|
| 67 |
+
max_repair_attempts: int,
|
| 68 |
+
) -> None:
|
| 69 |
+
self.datasets = datasets
|
| 70 |
+
self.sessions = sessions
|
| 71 |
+
self.llm = llm
|
| 72 |
+
self.executor = executor
|
| 73 |
+
self.max_repair_attempts = max_repair_attempts
|
| 74 |
+
|
| 75 |
+
def analyze(self, request: AnalysisRequest) -> AnalysisResponse:
|
| 76 |
+
stored = self.datasets.get(request.dataset_id)
|
| 77 |
+
session_id = request.session_id or self.sessions.create_session_id()
|
| 78 |
+
history = self.sessions.history(session_id, limit=5)
|
| 79 |
+
generated = self._generate_code(
|
| 80 |
+
question=request.question,
|
| 81 |
+
profile=stored.profile.model_dump(mode="json"),
|
| 82 |
+
history=[self._history_summary(record.response) for record in history],
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
profile = stored.profile.model_dump(mode="json")
|
| 86 |
+
execution = self._execute_generated(stored.dataframe, generated, profile)
|
| 87 |
+
attempts = 1
|
| 88 |
+
while not execution.success and attempts <= self.max_repair_attempts:
|
| 89 |
+
generated = self._repair_code(
|
| 90 |
+
question=request.question,
|
| 91 |
+
profile=profile,
|
| 92 |
+
failed_code=generated.code,
|
| 93 |
+
error=execution.error or "Unknown execution error.",
|
| 94 |
+
)
|
| 95 |
+
attempts += 1
|
| 96 |
+
execution = self._execute_generated(stored.dataframe, generated, profile)
|
| 97 |
+
execution.attempts = attempts
|
| 98 |
+
|
| 99 |
+
if not execution.success:
|
| 100 |
+
raise ExecutionAppError(
|
| 101 |
+
"analysis_execution_failed",
|
| 102 |
+
"Generated analysis code failed after repair attempts.",
|
| 103 |
+
status_code=422,
|
| 104 |
+
details={"attempts": attempts, "error": execution.error},
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
narrative = self._generate_narrative(
|
| 108 |
+
question=request.question,
|
| 109 |
+
profile=stored.profile.model_dump(mode="json"),
|
| 110 |
+
execution=compact_result_for_prompt(execution),
|
| 111 |
+
)
|
| 112 |
+
response = AnalysisResponse(
|
| 113 |
+
session_id=session_id,
|
| 114 |
+
dataset_id=request.dataset_id,
|
| 115 |
+
question=request.question,
|
| 116 |
+
plan=generated.plan,
|
| 117 |
+
generated_code=generated.code,
|
| 118 |
+
execution=execution,
|
| 119 |
+
narrative=narrative,
|
| 120 |
+
)
|
| 121 |
+
self.sessions.append(session_id, request.dataset_id, request.question, response)
|
| 122 |
+
return response
|
| 123 |
+
|
| 124 |
+
def _execute_generated(
|
| 125 |
+
self,
|
| 126 |
+
df,
|
| 127 |
+
generated: GeneratedAnalysisCode,
|
| 128 |
+
profile: dict,
|
| 129 |
+
) -> ExecutionResult:
|
| 130 |
+
plan_error = self._validate_generated_plan(generated, profile)
|
| 131 |
+
if plan_error is not None:
|
| 132 |
+
return plan_error
|
| 133 |
+
try:
|
| 134 |
+
return self.executor.execute(df, generated.code)
|
| 135 |
+
except ExecutionAppError as exc:
|
| 136 |
+
return ExecutionResult(
|
| 137 |
+
success=False,
|
| 138 |
+
error=f"{exc.code}: {exc.message}",
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
def _validate_generated_plan(
|
| 142 |
+
self,
|
| 143 |
+
generated: GeneratedAnalysisCode,
|
| 144 |
+
profile: dict,
|
| 145 |
+
) -> ExecutionResult | None:
|
| 146 |
+
available_columns = {str(column["name"]) for column in profile.get("columns", [])}
|
| 147 |
+
missing = [
|
| 148 |
+
column
|
| 149 |
+
for column in generated.plan.required_columns
|
| 150 |
+
if column not in available_columns
|
| 151 |
+
]
|
| 152 |
+
if missing:
|
| 153 |
+
return ExecutionResult(
|
| 154 |
+
success=False,
|
| 155 |
+
error=(
|
| 156 |
+
"Generated plan referenced columns that do not exist in the dataset: "
|
| 157 |
+
f"{missing}. Available columns: {sorted(available_columns)}"
|
| 158 |
+
),
|
| 159 |
+
)
|
| 160 |
+
return None
|
| 161 |
+
|
| 162 |
+
def _generate_code(
|
| 163 |
+
self,
|
| 164 |
+
*,
|
| 165 |
+
question: str,
|
| 166 |
+
profile: dict,
|
| 167 |
+
history: list[dict],
|
| 168 |
+
) -> GeneratedAnalysisCode:
|
| 169 |
+
user = json.dumps(
|
| 170 |
+
{"question": question, "dataset_profile": profile, "session_history": history},
|
| 171 |
+
ensure_ascii=True,
|
| 172 |
+
)
|
| 173 |
+
raw = self.llm.chat_json(system=GENERATION_SYSTEM_PROMPT, user=user)
|
| 174 |
+
return self._parse_generated(raw)
|
| 175 |
+
|
| 176 |
+
def _repair_code(
|
| 177 |
+
self,
|
| 178 |
+
*,
|
| 179 |
+
question: str,
|
| 180 |
+
profile: dict,
|
| 181 |
+
failed_code: str,
|
| 182 |
+
error: str,
|
| 183 |
+
) -> GeneratedAnalysisCode:
|
| 184 |
+
user = json.dumps(
|
| 185 |
+
{
|
| 186 |
+
"question": question,
|
| 187 |
+
"dataset_profile": profile,
|
| 188 |
+
"failed_code": failed_code,
|
| 189 |
+
"execution_error": error,
|
| 190 |
+
},
|
| 191 |
+
ensure_ascii=True,
|
| 192 |
+
)
|
| 193 |
+
raw = self.llm.chat_json(system=REPAIR_SYSTEM_PROMPT, user=user)
|
| 194 |
+
return self._parse_generated(raw)
|
| 195 |
+
|
| 196 |
+
def _generate_narrative(self, *, question: str, profile: dict, execution: dict) -> Narrative:
|
| 197 |
+
user = json.dumps(
|
| 198 |
+
{"question": question, "dataset_profile": profile, "execution_result": execution},
|
| 199 |
+
ensure_ascii=True,
|
| 200 |
+
)
|
| 201 |
+
raw = self.llm.chat_json(system=NARRATIVE_SYSTEM_PROMPT, user=user)
|
| 202 |
+
try:
|
| 203 |
+
return Narrative.model_validate(
|
| 204 |
+
self._normalize_narrative_payload(extract_json_object(raw))
|
| 205 |
+
)
|
| 206 |
+
except ValidationAppError:
|
| 207 |
+
raise
|
| 208 |
+
except Exception as exc:
|
| 209 |
+
raise ValidationAppError(
|
| 210 |
+
"invalid_narrative",
|
| 211 |
+
"The model returned a narrative that did not match the required schema.",
|
| 212 |
+
details={"reason": str(exc)},
|
| 213 |
+
) from exc
|
| 214 |
+
|
| 215 |
+
def _normalize_narrative_payload(self, data: dict) -> dict:
|
| 216 |
+
normalized = dict(data)
|
| 217 |
+
for key in ("limitations", "follow_up_questions"):
|
| 218 |
+
value = normalized.get(key)
|
| 219 |
+
if isinstance(value, str):
|
| 220 |
+
normalized[key] = [value]
|
| 221 |
+
elif value is None:
|
| 222 |
+
normalized[key] = []
|
| 223 |
+
for key in ("key_finding", "business_meaning"):
|
| 224 |
+
value = normalized.get(key)
|
| 225 |
+
if isinstance(value, list):
|
| 226 |
+
normalized[key] = " ".join(str(item) for item in value)
|
| 227 |
+
return normalized
|
| 228 |
+
|
| 229 |
+
def _parse_generated(self, raw: str) -> GeneratedAnalysisCode:
|
| 230 |
+
data = extract_json_object(raw)
|
| 231 |
+
data = self._normalize_generated_payload(data)
|
| 232 |
+
try:
|
| 233 |
+
return GeneratedAnalysisCode.model_validate(data)
|
| 234 |
+
except Exception as exc:
|
| 235 |
+
raise ValidationAppError(
|
| 236 |
+
"invalid_generated_analysis",
|
| 237 |
+
"The model returned generated analysis that did not match the required schema.",
|
| 238 |
+
details={"reason": str(exc)},
|
| 239 |
+
) from exc
|
| 240 |
+
|
| 241 |
+
def _normalize_generated_payload(self, data: dict) -> dict:
|
| 242 |
+
if "plan" in data and "code" in data:
|
| 243 |
+
return data
|
| 244 |
+
|
| 245 |
+
if "result_df" not in data and "chart_spec" not in data:
|
| 246 |
+
return data
|
| 247 |
+
|
| 248 |
+
result_expr = data.get("result_df", "df.head(10)")
|
| 249 |
+
if not isinstance(result_expr, str) or not result_expr.strip():
|
| 250 |
+
result_expr = "df.head(10)"
|
| 251 |
+
chart_spec = data.get("chart_spec")
|
| 252 |
+
if not isinstance(chart_spec, dict):
|
| 253 |
+
chart_spec = {
|
| 254 |
+
"chart_type": "table",
|
| 255 |
+
"title": "Dataset Overview",
|
| 256 |
+
"x": None,
|
| 257 |
+
"y": None,
|
| 258 |
+
"caption": "Generated dataset overview.",
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
chart_type = chart_spec.get("chart_type", ChartType.TABLE)
|
| 262 |
+
required_columns = [
|
| 263 |
+
value
|
| 264 |
+
for value in (chart_spec.get("x"), chart_spec.get("y"), chart_spec.get("color"))
|
| 265 |
+
if isinstance(value, str) and value
|
| 266 |
+
]
|
| 267 |
+
code = "\n".join(
|
| 268 |
+
[
|
| 269 |
+
f"result_df = {result_expr}",
|
| 270 |
+
f"chart_spec = {chart_spec!r}",
|
| 271 |
+
]
|
| 272 |
+
)
|
| 273 |
+
return {
|
| 274 |
+
"plan": AnalysisPlan(
|
| 275 |
+
operation=AnalysisOperation.OVERVIEW,
|
| 276 |
+
objective=str(
|
| 277 |
+
chart_spec.get("caption")
|
| 278 |
+
or chart_spec.get("title")
|
| 279 |
+
or "Dataset overview"
|
| 280 |
+
),
|
| 281 |
+
required_columns=required_columns,
|
| 282 |
+
assumptions=[
|
| 283 |
+
"Model returned a compact result expression and chart spec; "
|
| 284 |
+
"backend normalized it to the executable analysis contract."
|
| 285 |
+
],
|
| 286 |
+
chart_type=chart_type,
|
| 287 |
+
).model_dump(mode="json"),
|
| 288 |
+
"code": code,
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
def _history_summary(self, response: AnalysisResponse) -> dict:
|
| 292 |
+
return {
|
| 293 |
+
"question": response.question,
|
| 294 |
+
"operation": response.plan.operation,
|
| 295 |
+
"key_finding": response.narrative.key_finding,
|
| 296 |
+
"result_columns": response.execution.result_columns,
|
| 297 |
+
"sample_rows": response.execution.result_rows[:5],
|
| 298 |
+
}
|
src/autodata_agent/services/charts.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from typing import Any
|
| 5 |
+
|
| 6 |
+
import pandas as pd
|
| 7 |
+
import plotly.express as px
|
| 8 |
+
import plotly.graph_objects as go
|
| 9 |
+
|
| 10 |
+
from autodata_agent.core.errors import ValidationAppError
|
| 11 |
+
from autodata_agent.core.schemas import ChartSpec, ChartType
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def render_chart_figure(chart_spec: ChartSpec, result_rows: list[dict[str, Any]]) -> dict[str, Any]:
|
| 15 |
+
if not result_rows:
|
| 16 |
+
raise ValidationAppError(
|
| 17 |
+
"empty_chart_data",
|
| 18 |
+
"Cannot render a chart without result rows.",
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
df = pd.DataFrame(result_rows)
|
| 22 |
+
chart_type = chart_spec.chart_type
|
| 23 |
+
|
| 24 |
+
if chart_type == ChartType.TABLE:
|
| 25 |
+
figure = _table_figure(df, chart_spec)
|
| 26 |
+
elif chart_type == ChartType.BAR:
|
| 27 |
+
figure = _bar_figure(df, chart_spec)
|
| 28 |
+
elif chart_type == ChartType.LINE:
|
| 29 |
+
figure = _line_figure(df, chart_spec)
|
| 30 |
+
elif chart_type == ChartType.SCATTER:
|
| 31 |
+
figure = _scatter_figure(df, chart_spec)
|
| 32 |
+
elif chart_type == ChartType.HEATMAP:
|
| 33 |
+
figure = _heatmap_figure(df, chart_spec)
|
| 34 |
+
elif chart_type == ChartType.DISTRIBUTION:
|
| 35 |
+
figure = _distribution_figure(df, chart_spec)
|
| 36 |
+
else:
|
| 37 |
+
raise ValidationAppError(
|
| 38 |
+
"unsupported_chart_type",
|
| 39 |
+
f"Chart type '{chart_type}' is not supported for rendering.",
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
figure.update_layout(
|
| 43 |
+
title=chart_spec.title,
|
| 44 |
+
margin={"l": 40, "r": 40, "t": 60, "b": 40},
|
| 45 |
+
)
|
| 46 |
+
if chart_type != ChartType.TABLE:
|
| 47 |
+
figure.update_layout(
|
| 48 |
+
xaxis_title=chart_spec.x,
|
| 49 |
+
yaxis_title=chart_spec.y,
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
return json.loads(figure.to_json())
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _require_columns(
|
| 56 |
+
df: pd.DataFrame,
|
| 57 |
+
columns: list[str | None],
|
| 58 |
+
*,
|
| 59 |
+
chart_type: str,
|
| 60 |
+
) -> None:
|
| 61 |
+
required = [column for column in columns if column]
|
| 62 |
+
missing = [column for column in required if column not in df.columns]
|
| 63 |
+
if missing:
|
| 64 |
+
raise ValidationAppError(
|
| 65 |
+
"chart_columns_missing",
|
| 66 |
+
f"Result data is missing columns required for {chart_type} chart rendering.",
|
| 67 |
+
details={"missing_columns": missing, "available_columns": list(df.columns)},
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _value_column(df: pd.DataFrame, chart_spec: ChartSpec, exclude: set[str]) -> str:
|
| 72 |
+
if chart_spec.color and chart_spec.color in df.columns and chart_spec.color not in exclude:
|
| 73 |
+
return chart_spec.color
|
| 74 |
+
for column in df.columns:
|
| 75 |
+
if column in exclude:
|
| 76 |
+
continue
|
| 77 |
+
if pd.api.types.is_numeric_dtype(df[column]):
|
| 78 |
+
return str(column)
|
| 79 |
+
raise ValidationAppError(
|
| 80 |
+
"chart_value_column_missing",
|
| 81 |
+
"Could not determine a numeric value column for chart rendering.",
|
| 82 |
+
details={"available_columns": list(df.columns)},
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _distribution_column(df: pd.DataFrame, chart_spec: ChartSpec) -> str:
|
| 87 |
+
for column in (chart_spec.x, chart_spec.y):
|
| 88 |
+
if column and column in df.columns:
|
| 89 |
+
return column
|
| 90 |
+
for column in df.columns:
|
| 91 |
+
if pd.api.types.is_numeric_dtype(df[column]):
|
| 92 |
+
return str(column)
|
| 93 |
+
raise ValidationAppError(
|
| 94 |
+
"chart_distribution_column_missing",
|
| 95 |
+
"Could not determine a column for distribution chart rendering.",
|
| 96 |
+
details={"available_columns": list(df.columns)},
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _bar_figure(df: pd.DataFrame, chart_spec: ChartSpec) -> go.Figure:
|
| 101 |
+
_require_columns(df, [chart_spec.x, chart_spec.y], chart_type="bar")
|
| 102 |
+
color = chart_spec.color if chart_spec.color and chart_spec.color in df.columns else None
|
| 103 |
+
return px.bar(df, x=chart_spec.x, y=chart_spec.y, color=color)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _line_figure(df: pd.DataFrame, chart_spec: ChartSpec) -> go.Figure:
|
| 107 |
+
_require_columns(df, [chart_spec.x, chart_spec.y], chart_type="line")
|
| 108 |
+
color = chart_spec.color if chart_spec.color and chart_spec.color in df.columns else None
|
| 109 |
+
return px.line(df, x=chart_spec.x, y=chart_spec.y, color=color)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _scatter_figure(df: pd.DataFrame, chart_spec: ChartSpec) -> go.Figure:
|
| 113 |
+
_require_columns(df, [chart_spec.x, chart_spec.y], chart_type="scatter")
|
| 114 |
+
color = chart_spec.color if chart_spec.color and chart_spec.color in df.columns else None
|
| 115 |
+
return px.scatter(df, x=chart_spec.x, y=chart_spec.y, color=color)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _distribution_figure(df: pd.DataFrame, chart_spec: ChartSpec) -> go.Figure:
|
| 119 |
+
column = _distribution_column(df, chart_spec)
|
| 120 |
+
return px.histogram(df, x=column)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _heatmap_figure(df: pd.DataFrame, chart_spec: ChartSpec) -> go.Figure:
|
| 124 |
+
_require_columns(df, [chart_spec.x, chart_spec.y], chart_type="heatmap")
|
| 125 |
+
value_column = _value_column(
|
| 126 |
+
df,
|
| 127 |
+
chart_spec,
|
| 128 |
+
exclude={chart_spec.x or "", chart_spec.y or ""},
|
| 129 |
+
)
|
| 130 |
+
pivot = df.pivot_table(
|
| 131 |
+
index=chart_spec.x,
|
| 132 |
+
columns=chart_spec.y,
|
| 133 |
+
values=value_column,
|
| 134 |
+
aggfunc="mean",
|
| 135 |
+
)
|
| 136 |
+
return px.imshow(
|
| 137 |
+
pivot,
|
| 138 |
+
labels={"x": chart_spec.y, "y": chart_spec.x, "color": value_column},
|
| 139 |
+
aspect="auto",
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _table_figure(df: pd.DataFrame, chart_spec: ChartSpec) -> go.Figure:
|
| 144 |
+
return go.Figure(
|
| 145 |
+
data=[
|
| 146 |
+
go.Table(
|
| 147 |
+
header=dict(values=[str(column) for column in df.columns]),
|
| 148 |
+
cells=dict(values=[df[column].tolist() for column in df.columns]),
|
| 149 |
+
)
|
| 150 |
+
]
|
| 151 |
+
)
|
src/autodata_agent/services/datasets.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import io
|
| 4 |
+
import json
|
| 5 |
+
import uuid
|
| 6 |
+
from dataclasses import dataclass
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import pandas as pd
|
| 10 |
+
from sqlalchemy import create_engine, text
|
| 11 |
+
|
| 12 |
+
from autodata_agent.core.errors import ValidationAppError
|
| 13 |
+
from autodata_agent.core.schemas import (
|
| 14 |
+
DatasetInfo,
|
| 15 |
+
DatasetPreview,
|
| 16 |
+
DatasetProfile,
|
| 17 |
+
DataSourceType,
|
| 18 |
+
SQLIngestRequest,
|
| 19 |
+
)
|
| 20 |
+
from autodata_agent.services.profiling import build_profile
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class StoredDataset:
|
| 25 |
+
dataset_id: str
|
| 26 |
+
dataframe: pd.DataFrame
|
| 27 |
+
source_name: str
|
| 28 |
+
source_type: DataSourceType
|
| 29 |
+
profile: DatasetProfile
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class DatasetStore:
|
| 33 |
+
def __init__(
|
| 34 |
+
self,
|
| 35 |
+
upload_dir: Path,
|
| 36 |
+
max_upload_bytes: int,
|
| 37 |
+
dataset_dir: Path | None = None,
|
| 38 |
+
) -> None:
|
| 39 |
+
self.upload_dir = upload_dir
|
| 40 |
+
self.dataset_dir = dataset_dir or upload_dir.parent / "datasets"
|
| 41 |
+
self.max_upload_bytes = max_upload_bytes
|
| 42 |
+
self._datasets: dict[str, StoredDataset] = {}
|
| 43 |
+
self.upload_dir.mkdir(parents=True, exist_ok=True)
|
| 44 |
+
self.dataset_dir.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
|
| 46 |
+
def put_file(self, filename: str, content: bytes) -> DatasetInfo:
|
| 47 |
+
if not content:
|
| 48 |
+
raise ValidationAppError("empty_upload", "Uploaded file is empty.")
|
| 49 |
+
if len(content) > self.max_upload_bytes:
|
| 50 |
+
raise ValidationAppError(
|
| 51 |
+
"upload_too_large",
|
| 52 |
+
"Uploaded file exceeds the configured size limit.",
|
| 53 |
+
details={"max_upload_bytes": self.max_upload_bytes},
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
suffix = Path(filename).suffix.lower()
|
| 57 |
+
if suffix == ".csv":
|
| 58 |
+
source_type = DataSourceType.CSV
|
| 59 |
+
df = pd.read_csv(io.BytesIO(content))
|
| 60 |
+
elif suffix in {".xlsx", ".xls"}:
|
| 61 |
+
source_type = DataSourceType.EXCEL
|
| 62 |
+
df = pd.read_excel(io.BytesIO(content))
|
| 63 |
+
elif suffix == ".json":
|
| 64 |
+
source_type = DataSourceType.JSON
|
| 65 |
+
df = pd.read_json(io.BytesIO(content))
|
| 66 |
+
else:
|
| 67 |
+
raise ValidationAppError(
|
| 68 |
+
"unsupported_file_type",
|
| 69 |
+
"Supported file types are CSV, Excel, and JSON.",
|
| 70 |
+
details={"filename": filename},
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
saved_name = f"{uuid.uuid4().hex}_{Path(filename).name}"
|
| 74 |
+
self.upload_dir.mkdir(parents=True, exist_ok=True)
|
| 75 |
+
(self.upload_dir / saved_name).write_bytes(content)
|
| 76 |
+
return self._store(df, Path(filename).name, source_type)
|
| 77 |
+
|
| 78 |
+
def put_sql(self, request: SQLIngestRequest) -> DatasetInfo:
|
| 79 |
+
cleaned = request.query.strip().lower()
|
| 80 |
+
if not cleaned.startswith("select"):
|
| 81 |
+
raise ValidationAppError(
|
| 82 |
+
"unsafe_sql_query",
|
| 83 |
+
"Only SELECT queries are allowed for SQL ingestion.",
|
| 84 |
+
)
|
| 85 |
+
try:
|
| 86 |
+
engine = create_engine(request.connection_uri)
|
| 87 |
+
with engine.connect() as connection:
|
| 88 |
+
df = pd.read_sql_query(text(request.query), connection)
|
| 89 |
+
except Exception as exc: # noqa: BLE001 - returned as structured API error
|
| 90 |
+
raise ValidationAppError(
|
| 91 |
+
"sql_ingestion_failed",
|
| 92 |
+
"SQL ingestion failed.",
|
| 93 |
+
details={"reason": str(exc)},
|
| 94 |
+
) from exc
|
| 95 |
+
return self._store(df, request.source_name, DataSourceType.SQL)
|
| 96 |
+
|
| 97 |
+
def get(self, dataset_id: str) -> StoredDataset:
|
| 98 |
+
try:
|
| 99 |
+
return self._datasets[dataset_id]
|
| 100 |
+
except KeyError as exc:
|
| 101 |
+
rehydrated = self._rehydrate(dataset_id)
|
| 102 |
+
if rehydrated is not None:
|
| 103 |
+
self._datasets[dataset_id] = rehydrated
|
| 104 |
+
return rehydrated
|
| 105 |
+
raise ValidationAppError(
|
| 106 |
+
"dataset_not_found",
|
| 107 |
+
"Dataset was not found in backend storage.",
|
| 108 |
+
status_code=404,
|
| 109 |
+
details={"dataset_id": dataset_id},
|
| 110 |
+
) from exc
|
| 111 |
+
|
| 112 |
+
def preview(self, dataset_id: str, limit: int = 20) -> DatasetPreview:
|
| 113 |
+
stored = self.get(dataset_id)
|
| 114 |
+
df = stored.dataframe.head(limit)
|
| 115 |
+
rows = json.loads(
|
| 116 |
+
df.where(pd.notnull(df), None).to_json(orient="records", date_format="iso")
|
| 117 |
+
)
|
| 118 |
+
return DatasetPreview(
|
| 119 |
+
dataset_id=dataset_id,
|
| 120 |
+
columns=[str(col) for col in df.columns],
|
| 121 |
+
rows=rows,
|
| 122 |
+
row_count=len(stored.dataframe),
|
| 123 |
+
returned_rows=len(rows),
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
def _store(
|
| 127 |
+
self,
|
| 128 |
+
df: pd.DataFrame,
|
| 129 |
+
source_name: str,
|
| 130 |
+
source_type: DataSourceType,
|
| 131 |
+
) -> DatasetInfo:
|
| 132 |
+
if df.empty:
|
| 133 |
+
raise ValidationAppError("empty_dataset", "Dataset contains no rows.")
|
| 134 |
+
df = df.copy()
|
| 135 |
+
df.columns = [str(column).strip() for column in df.columns]
|
| 136 |
+
if not all(df.columns):
|
| 137 |
+
raise ValidationAppError("invalid_columns", "Dataset contains blank column names.")
|
| 138 |
+
dataset_id = uuid.uuid4().hex
|
| 139 |
+
profile = build_profile(
|
| 140 |
+
df,
|
| 141 |
+
dataset_id=dataset_id,
|
| 142 |
+
source_name=source_name,
|
| 143 |
+
source_type=source_type,
|
| 144 |
+
)
|
| 145 |
+
self._datasets[dataset_id] = StoredDataset(
|
| 146 |
+
dataset_id,
|
| 147 |
+
df,
|
| 148 |
+
source_name,
|
| 149 |
+
source_type,
|
| 150 |
+
profile,
|
| 151 |
+
)
|
| 152 |
+
self._persist(dataset_id, df, profile)
|
| 153 |
+
return DatasetInfo(
|
| 154 |
+
dataset_id=dataset_id,
|
| 155 |
+
source_name=source_name,
|
| 156 |
+
source_type=source_type,
|
| 157 |
+
row_count=profile.row_count,
|
| 158 |
+
column_count=profile.column_count,
|
| 159 |
+
profile=profile,
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
def _persist(self, dataset_id: str, df: pd.DataFrame, profile: DatasetProfile) -> None:
|
| 163 |
+
dataset_path = self.dataset_dir / f"{dataset_id}.csv"
|
| 164 |
+
profile_path = self.dataset_dir / f"{dataset_id}.profile.json"
|
| 165 |
+
df.to_csv(dataset_path, index=False)
|
| 166 |
+
profile_path.write_text(profile.model_dump_json(indent=2), encoding="utf-8")
|
| 167 |
+
|
| 168 |
+
def _rehydrate(self, dataset_id: str) -> StoredDataset | None:
|
| 169 |
+
dataset_path = self.dataset_dir / f"{dataset_id}.csv"
|
| 170 |
+
profile_path = self.dataset_dir / f"{dataset_id}.profile.json"
|
| 171 |
+
if not dataset_path.exists() or not profile_path.exists():
|
| 172 |
+
return None
|
| 173 |
+
df = pd.read_csv(dataset_path)
|
| 174 |
+
profile = DatasetProfile.model_validate_json(profile_path.read_text(encoding="utf-8"))
|
| 175 |
+
return StoredDataset(
|
| 176 |
+
dataset_id=dataset_id,
|
| 177 |
+
dataframe=df,
|
| 178 |
+
source_name=profile.source_name,
|
| 179 |
+
source_type=profile.source_type,
|
| 180 |
+
profile=profile,
|
| 181 |
+
)
|
src/autodata_agent/services/executor.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import subprocess
|
| 5 |
+
import sys
|
| 6 |
+
import tempfile
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
import pandas as pd
|
| 11 |
+
|
| 12 |
+
from autodata_agent.core.errors import ExecutionAppError
|
| 13 |
+
from autodata_agent.core.schemas import ChartSpec, ExecutionResult
|
| 14 |
+
|
| 15 |
+
BANNED_TOKENS = (
|
| 16 |
+
"import ",
|
| 17 |
+
"__import__",
|
| 18 |
+
"open(",
|
| 19 |
+
"exec(",
|
| 20 |
+
"eval(",
|
| 21 |
+
"compile(",
|
| 22 |
+
"subprocess",
|
| 23 |
+
"socket",
|
| 24 |
+
"requests",
|
| 25 |
+
"httpx",
|
| 26 |
+
"urllib",
|
| 27 |
+
"pathlib",
|
| 28 |
+
"os.",
|
| 29 |
+
"sys.",
|
| 30 |
+
"shutil",
|
| 31 |
+
"pickle",
|
| 32 |
+
"globals(",
|
| 33 |
+
"locals(",
|
| 34 |
+
"__",
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
RUNNER_CODE = r"""
|
| 39 |
+
import contextlib
|
| 40 |
+
import io
|
| 41 |
+
import json
|
| 42 |
+
import math
|
| 43 |
+
import statistics
|
| 44 |
+
import traceback
|
| 45 |
+
|
| 46 |
+
import duckdb
|
| 47 |
+
import numpy as np
|
| 48 |
+
import pandas as pd
|
| 49 |
+
|
| 50 |
+
with open(INPUT_JSON, "r", encoding="utf-8") as handle:
|
| 51 |
+
payload = json.load(handle)
|
| 52 |
+
|
| 53 |
+
df = pd.DataFrame(payload["records"])
|
| 54 |
+
code = payload["code"]
|
| 55 |
+
|
| 56 |
+
allowed_builtins = {
|
| 57 |
+
"abs": abs,
|
| 58 |
+
"all": all,
|
| 59 |
+
"any": any,
|
| 60 |
+
"bool": bool,
|
| 61 |
+
"dict": dict,
|
| 62 |
+
"enumerate": enumerate,
|
| 63 |
+
"float": float,
|
| 64 |
+
"int": int,
|
| 65 |
+
"len": len,
|
| 66 |
+
"list": list,
|
| 67 |
+
"max": max,
|
| 68 |
+
"min": min,
|
| 69 |
+
"pow": pow,
|
| 70 |
+
"range": range,
|
| 71 |
+
"round": round,
|
| 72 |
+
"set": set,
|
| 73 |
+
"sorted": sorted,
|
| 74 |
+
"str": str,
|
| 75 |
+
"sum": sum,
|
| 76 |
+
"tuple": tuple,
|
| 77 |
+
"zip": zip,
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
scope = {
|
| 81 |
+
"__builtins__": allowed_builtins,
|
| 82 |
+
"df": df,
|
| 83 |
+
"pd": pd,
|
| 84 |
+
"np": np,
|
| 85 |
+
"duckdb": duckdb,
|
| 86 |
+
"math": math,
|
| 87 |
+
"statistics": statistics,
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
stdout = io.StringIO()
|
| 91 |
+
try:
|
| 92 |
+
with contextlib.redirect_stdout(stdout):
|
| 93 |
+
exec(code, scope, scope)
|
| 94 |
+
result_df = scope.get("result_df")
|
| 95 |
+
chart_spec = scope.get("chart_spec")
|
| 96 |
+
if result_df is None:
|
| 97 |
+
raise ValueError("Generated code must define result_df.")
|
| 98 |
+
if not isinstance(result_df, pd.DataFrame):
|
| 99 |
+
result_df = pd.DataFrame(result_df)
|
| 100 |
+
if chart_spec is None:
|
| 101 |
+
raise ValueError("Generated code must define chart_spec.")
|
| 102 |
+
if not isinstance(chart_spec, dict):
|
| 103 |
+
raise ValueError("chart_spec must be a dictionary.")
|
| 104 |
+
result_df = result_df.head(200)
|
| 105 |
+
output = {
|
| 106 |
+
"success": True,
|
| 107 |
+
"result_columns": [str(column) for column in result_df.columns],
|
| 108 |
+
"result_rows": json.loads(
|
| 109 |
+
result_df.where(pd.notnull(result_df), None).to_json(orient="records")
|
| 110 |
+
),
|
| 111 |
+
"chart_spec": chart_spec,
|
| 112 |
+
"stdout": stdout.getvalue(),
|
| 113 |
+
}
|
| 114 |
+
except Exception:
|
| 115 |
+
output = {
|
| 116 |
+
"success": False,
|
| 117 |
+
"error": traceback.format_exc(limit=8),
|
| 118 |
+
"stdout": stdout.getvalue(),
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
with open(OUTPUT_JSON, "w", encoding="utf-8") as handle:
|
| 122 |
+
json.dump(output, handle, ensure_ascii=True)
|
| 123 |
+
"""
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
class CodeExecutor:
|
| 127 |
+
def __init__(self, timeout_seconds: float) -> None:
|
| 128 |
+
self.timeout_seconds = timeout_seconds
|
| 129 |
+
|
| 130 |
+
def execute(self, df: pd.DataFrame, code: str) -> ExecutionResult:
|
| 131 |
+
self._validate_code(code)
|
| 132 |
+
records = json.loads(
|
| 133 |
+
df.where(pd.notnull(df), None).to_json(orient="records", date_format="iso")
|
| 134 |
+
)
|
| 135 |
+
with tempfile.TemporaryDirectory(prefix="autodata_exec_") as tmp:
|
| 136 |
+
tmp_path = Path(tmp)
|
| 137 |
+
input_path = tmp_path / "input.json"
|
| 138 |
+
output_path = tmp_path / "output.json"
|
| 139 |
+
runner_path = tmp_path / "runner.py"
|
| 140 |
+
input_path.write_text(json.dumps({"records": records, "code": code}), encoding="utf-8")
|
| 141 |
+
runner_path.write_text(
|
| 142 |
+
"INPUT_JSON = " + repr(str(input_path)) + "\n"
|
| 143 |
+
"OUTPUT_JSON = " + repr(str(output_path)) + "\n"
|
| 144 |
+
+ RUNNER_CODE,
|
| 145 |
+
encoding="utf-8",
|
| 146 |
+
)
|
| 147 |
+
try:
|
| 148 |
+
subprocess.run(
|
| 149 |
+
[sys.executable, str(runner_path)],
|
| 150 |
+
capture_output=True,
|
| 151 |
+
text=True,
|
| 152 |
+
timeout=self.timeout_seconds,
|
| 153 |
+
check=False,
|
| 154 |
+
)
|
| 155 |
+
except subprocess.TimeoutExpired as exc:
|
| 156 |
+
raise ExecutionAppError(
|
| 157 |
+
"analysis_timeout",
|
| 158 |
+
"Generated analysis code exceeded the execution timeout.",
|
| 159 |
+
details={"timeout_seconds": self.timeout_seconds},
|
| 160 |
+
) from exc
|
| 161 |
+
|
| 162 |
+
if not output_path.exists():
|
| 163 |
+
raise ExecutionAppError(
|
| 164 |
+
"analysis_runner_failed",
|
| 165 |
+
"Analysis runner did not return a result.",
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
payload = json.loads(output_path.read_text(encoding="utf-8"))
|
| 169 |
+
if payload.get("success"):
|
| 170 |
+
chart = ChartSpec.model_validate(payload["chart_spec"])
|
| 171 |
+
return ExecutionResult(
|
| 172 |
+
success=True,
|
| 173 |
+
result_columns=payload.get("result_columns", []),
|
| 174 |
+
result_rows=payload.get("result_rows", []),
|
| 175 |
+
chart_spec=chart,
|
| 176 |
+
stdout=payload.get("stdout", ""),
|
| 177 |
+
)
|
| 178 |
+
return ExecutionResult(
|
| 179 |
+
success=False,
|
| 180 |
+
error=payload.get("error", "Unknown execution error."),
|
| 181 |
+
stdout=payload.get("stdout", ""),
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
def _validate_code(self, code: str) -> None:
|
| 185 |
+
lowered = code.lower()
|
| 186 |
+
for token in BANNED_TOKENS:
|
| 187 |
+
if token in lowered:
|
| 188 |
+
raise ExecutionAppError(
|
| 189 |
+
"unsafe_generated_code",
|
| 190 |
+
"Generated code contains a blocked operation.",
|
| 191 |
+
details={"blocked_token": token.strip()},
|
| 192 |
+
)
|
| 193 |
+
if len(code) > 20_000:
|
| 194 |
+
raise ExecutionAppError(
|
| 195 |
+
"generated_code_too_large",
|
| 196 |
+
"Generated code is too large to execute safely.",
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def compact_result_for_prompt(result: ExecutionResult) -> dict[str, Any]:
|
| 201 |
+
return {
|
| 202 |
+
"success": result.success,
|
| 203 |
+
"columns": result.result_columns,
|
| 204 |
+
"sample_rows": result.result_rows[:10],
|
| 205 |
+
"chart_spec": result.chart_spec.model_dump() if result.chart_spec else None,
|
| 206 |
+
"error": result.error,
|
| 207 |
+
}
|
src/autodata_agent/services/llm.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Protocol
|
| 4 |
+
|
| 5 |
+
import httpx
|
| 6 |
+
|
| 7 |
+
from autodata_agent.core.config import Settings
|
| 8 |
+
from autodata_agent.core.errors import ExternalServiceError
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class LLMClient(Protocol):
|
| 12 |
+
def chat_json(self, *, system: str, user: str) -> str:
|
| 13 |
+
"""Return model text that should contain a JSON object."""
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class OllamaCloudClient:
|
| 17 |
+
def __init__(self, settings: Settings) -> None:
|
| 18 |
+
self.settings = settings
|
| 19 |
+
|
| 20 |
+
def chat_json(self, *, system: str, user: str) -> str:
|
| 21 |
+
api_key = self.settings.ollama_api_key
|
| 22 |
+
if api_key is None or not api_key.get_secret_value().strip():
|
| 23 |
+
raise ExternalServiceError(
|
| 24 |
+
"ollama_api_key_missing",
|
| 25 |
+
"Ollama API key is not configured. Set AUTODATA_OLLAMA_API_KEY.",
|
| 26 |
+
status_code=503,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
url = self.settings.ollama_base_url.rstrip("/") + "/api/chat"
|
| 30 |
+
payload = {
|
| 31 |
+
"model": self.settings.ollama_model,
|
| 32 |
+
"messages": [
|
| 33 |
+
{"role": "system", "content": system},
|
| 34 |
+
{"role": "user", "content": user},
|
| 35 |
+
],
|
| 36 |
+
"format": "json",
|
| 37 |
+
"options": {
|
| 38 |
+
"temperature": 0.1,
|
| 39 |
+
"num_predict": self.settings.ollama_num_predict,
|
| 40 |
+
},
|
| 41 |
+
"stream": False,
|
| 42 |
+
}
|
| 43 |
+
if self.settings.ollama_think.strip().lower() not in {"", "0", "false", "off", "none"}:
|
| 44 |
+
payload["think"] = self.settings.ollama_think
|
| 45 |
+
headers = {"Authorization": f"Bearer {api_key.get_secret_value()}"}
|
| 46 |
+
try:
|
| 47 |
+
response = httpx.post(
|
| 48 |
+
url,
|
| 49 |
+
json=payload,
|
| 50 |
+
headers=headers,
|
| 51 |
+
timeout=self.settings.llm_timeout_seconds,
|
| 52 |
+
)
|
| 53 |
+
response.raise_for_status()
|
| 54 |
+
data = response.json()
|
| 55 |
+
return data["message"]["content"]
|
| 56 |
+
except httpx.HTTPStatusError as exc:
|
| 57 |
+
ollama_error = _extract_ollama_error(exc.response)
|
| 58 |
+
raise ExternalServiceError(
|
| 59 |
+
"ollama_request_failed",
|
| 60 |
+
ollama_error or "Ollama model request failed.",
|
| 61 |
+
status_code=503,
|
| 62 |
+
details={
|
| 63 |
+
"reason": str(exc),
|
| 64 |
+
"status_code": exc.response.status_code,
|
| 65 |
+
"response_body": exc.response.text[:1000],
|
| 66 |
+
"ollama_error": ollama_error,
|
| 67 |
+
"model": self.settings.ollama_model,
|
| 68 |
+
"url": url,
|
| 69 |
+
},
|
| 70 |
+
) from exc
|
| 71 |
+
except (httpx.HTTPError, KeyError, TypeError, ValueError) as exc:
|
| 72 |
+
raise ExternalServiceError(
|
| 73 |
+
"ollama_request_failed",
|
| 74 |
+
"Ollama model request failed.",
|
| 75 |
+
status_code=503,
|
| 76 |
+
details={
|
| 77 |
+
"reason": str(exc),
|
| 78 |
+
"model": self.settings.ollama_model,
|
| 79 |
+
"url": url,
|
| 80 |
+
},
|
| 81 |
+
) from exc
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _extract_ollama_error(response: httpx.Response) -> str | None:
|
| 85 |
+
try:
|
| 86 |
+
payload = response.json()
|
| 87 |
+
except ValueError:
|
| 88 |
+
return None
|
| 89 |
+
error = payload.get("error") if isinstance(payload, dict) else None
|
| 90 |
+
return error if isinstance(error, str) and error else None
|
src/autodata_agent/services/profiling.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import pandas as pd
|
| 7 |
+
|
| 8 |
+
from autodata_agent.core.schemas import ColumnProfile, DatasetProfile, DataSourceType
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _json_safe(value: Any) -> Any:
|
| 12 |
+
if pd.isna(value):
|
| 13 |
+
return None
|
| 14 |
+
if isinstance(value, np.generic):
|
| 15 |
+
return value.item()
|
| 16 |
+
if isinstance(value, pd.Timestamp):
|
| 17 |
+
return value.isoformat()
|
| 18 |
+
return value
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _numeric_summary(series: pd.Series) -> dict[str, Any]:
|
| 22 |
+
clean = pd.to_numeric(series, errors="coerce").dropna()
|
| 23 |
+
if clean.empty:
|
| 24 |
+
return {}
|
| 25 |
+
q1 = clean.quantile(0.25)
|
| 26 |
+
q3 = clean.quantile(0.75)
|
| 27 |
+
return {
|
| 28 |
+
"min": _json_safe(clean.min()),
|
| 29 |
+
"mean": _json_safe(clean.mean()),
|
| 30 |
+
"median": _json_safe(clean.median()),
|
| 31 |
+
"max": _json_safe(clean.max()),
|
| 32 |
+
"std": _json_safe(clean.std(ddof=0)),
|
| 33 |
+
"q1": _json_safe(q1),
|
| 34 |
+
"q3": _json_safe(q3),
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _categorical_summary(series: pd.Series) -> dict[str, Any]:
|
| 39 |
+
counts = series.dropna().astype(str).value_counts().head(10)
|
| 40 |
+
return {"top_values": [{"value": idx, "count": int(value)} for idx, value in counts.items()]}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _datetime_summary(series: pd.Series) -> dict[str, Any]:
|
| 44 |
+
dates = pd.to_datetime(series, errors="coerce").dropna()
|
| 45 |
+
if dates.empty:
|
| 46 |
+
return {}
|
| 47 |
+
return {"min": dates.min().isoformat(), "max": dates.max().isoformat()}
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def build_profile(
|
| 51 |
+
df: pd.DataFrame,
|
| 52 |
+
*,
|
| 53 |
+
dataset_id: str,
|
| 54 |
+
source_name: str,
|
| 55 |
+
source_type: DataSourceType,
|
| 56 |
+
) -> DatasetProfile:
|
| 57 |
+
row_count = int(len(df))
|
| 58 |
+
duplicate_rows = int(df.duplicated().sum()) if row_count else 0
|
| 59 |
+
columns: list[ColumnProfile] = []
|
| 60 |
+
anomalies: list[str] = []
|
| 61 |
+
|
| 62 |
+
if row_count == 0:
|
| 63 |
+
anomalies.append("Dataset has zero rows.")
|
| 64 |
+
|
| 65 |
+
if duplicate_rows:
|
| 66 |
+
anomalies.append(f"Dataset contains {duplicate_rows} duplicate rows.")
|
| 67 |
+
|
| 68 |
+
for name in df.columns:
|
| 69 |
+
series = df[name]
|
| 70 |
+
null_count = int(series.isna().sum())
|
| 71 |
+
null_rate = float(null_count / row_count) if row_count else 0.0
|
| 72 |
+
non_null = int(row_count - null_count)
|
| 73 |
+
unique_count = int(series.nunique(dropna=True))
|
| 74 |
+
dtype = str(series.dtype)
|
| 75 |
+
|
| 76 |
+
if null_rate >= 0.3:
|
| 77 |
+
anomalies.append(f"Column '{name}' has high missingness ({null_rate:.1%}).")
|
| 78 |
+
if row_count and unique_count == row_count and not pd.api.types.is_numeric_dtype(series):
|
| 79 |
+
anomalies.append(f"Column '{name}' appears to be a row identifier.")
|
| 80 |
+
|
| 81 |
+
summary: dict[str, Any]
|
| 82 |
+
if pd.api.types.is_numeric_dtype(series):
|
| 83 |
+
summary = _numeric_summary(series)
|
| 84 |
+
clean = pd.to_numeric(series, errors="coerce").dropna()
|
| 85 |
+
if len(clean) >= 8:
|
| 86 |
+
q1 = clean.quantile(0.25)
|
| 87 |
+
q3 = clean.quantile(0.75)
|
| 88 |
+
iqr = q3 - q1
|
| 89 |
+
if iqr > 0:
|
| 90 |
+
outlier_count = int(((clean < q1 - 1.5 * iqr) | (clean > q3 + 1.5 * iqr)).sum())
|
| 91 |
+
if outlier_count:
|
| 92 |
+
summary["iqr_outliers"] = outlier_count
|
| 93 |
+
anomalies.append(f"Column '{name}' has {outlier_count} IQR outliers.")
|
| 94 |
+
elif pd.api.types.is_datetime64_any_dtype(series):
|
| 95 |
+
summary = _datetime_summary(series)
|
| 96 |
+
else:
|
| 97 |
+
parsed_dates = pd.to_datetime(series, errors="coerce", format="mixed")
|
| 98 |
+
if parsed_dates.notna().mean() >= 0.8:
|
| 99 |
+
summary = _datetime_summary(series)
|
| 100 |
+
summary["inferred_type"] = "datetime"
|
| 101 |
+
else:
|
| 102 |
+
summary = _categorical_summary(series)
|
| 103 |
+
|
| 104 |
+
sample_values = [_json_safe(value) for value in series.dropna().head(5).tolist()]
|
| 105 |
+
columns.append(
|
| 106 |
+
ColumnProfile(
|
| 107 |
+
name=str(name),
|
| 108 |
+
dtype=dtype,
|
| 109 |
+
non_null_count=non_null,
|
| 110 |
+
null_count=null_count,
|
| 111 |
+
null_rate=round(null_rate, 4),
|
| 112 |
+
unique_count=unique_count,
|
| 113 |
+
sample_values=sample_values,
|
| 114 |
+
summary=summary,
|
| 115 |
+
)
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
return DatasetProfile(
|
| 119 |
+
dataset_id=dataset_id,
|
| 120 |
+
source_name=source_name,
|
| 121 |
+
source_type=source_type,
|
| 122 |
+
row_count=row_count,
|
| 123 |
+
column_count=int(len(df.columns)),
|
| 124 |
+
duplicate_rows=duplicate_rows,
|
| 125 |
+
columns=columns,
|
| 126 |
+
anomalies=anomalies,
|
| 127 |
+
)
|
src/autodata_agent/storage/session_store.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import sqlite3
|
| 5 |
+
import uuid
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
from autodata_agent.core.schemas import AnalysisResponse, SessionRecord
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class SessionStore:
|
| 12 |
+
def __init__(self, db_path: Path) -> None:
|
| 13 |
+
self.db_path = db_path
|
| 14 |
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
| 15 |
+
self._init_db()
|
| 16 |
+
|
| 17 |
+
def create_session_id(self) -> str:
|
| 18 |
+
return uuid.uuid4().hex
|
| 19 |
+
|
| 20 |
+
def append(
|
| 21 |
+
self,
|
| 22 |
+
session_id: str,
|
| 23 |
+
dataset_id: str,
|
| 24 |
+
question: str,
|
| 25 |
+
response: AnalysisResponse,
|
| 26 |
+
) -> str:
|
| 27 |
+
record_id = uuid.uuid4().hex
|
| 28 |
+
payload = response.model_dump_json()
|
| 29 |
+
with sqlite3.connect(self.db_path) as conn:
|
| 30 |
+
conn.execute(
|
| 31 |
+
"""
|
| 32 |
+
INSERT INTO session_records(
|
| 33 |
+
record_id,
|
| 34 |
+
session_id,
|
| 35 |
+
dataset_id,
|
| 36 |
+
question,
|
| 37 |
+
response_json
|
| 38 |
+
)
|
| 39 |
+
VALUES (?, ?, ?, ?, ?)
|
| 40 |
+
""",
|
| 41 |
+
(record_id, session_id, dataset_id, question, payload),
|
| 42 |
+
)
|
| 43 |
+
return record_id
|
| 44 |
+
|
| 45 |
+
def history(self, session_id: str, *, limit: int = 20) -> list[SessionRecord]:
|
| 46 |
+
with sqlite3.connect(self.db_path) as conn:
|
| 47 |
+
rows = conn.execute(
|
| 48 |
+
"""
|
| 49 |
+
SELECT record_id, session_id, dataset_id, question, response_json, created_at
|
| 50 |
+
FROM session_records
|
| 51 |
+
WHERE session_id = ?
|
| 52 |
+
ORDER BY id DESC
|
| 53 |
+
LIMIT ?
|
| 54 |
+
""",
|
| 55 |
+
(session_id, limit),
|
| 56 |
+
).fetchall()
|
| 57 |
+
records: list[SessionRecord] = []
|
| 58 |
+
for record_id, sid, dataset_id, question, response_json, created_at in reversed(rows):
|
| 59 |
+
records.append(
|
| 60 |
+
SessionRecord(
|
| 61 |
+
record_id=record_id,
|
| 62 |
+
session_id=sid,
|
| 63 |
+
dataset_id=dataset_id,
|
| 64 |
+
question=question,
|
| 65 |
+
response=AnalysisResponse.model_validate(json.loads(response_json)),
|
| 66 |
+
created_at=created_at,
|
| 67 |
+
)
|
| 68 |
+
)
|
| 69 |
+
return records
|
| 70 |
+
|
| 71 |
+
def all_records(self, session_id: str) -> list[SessionRecord]:
|
| 72 |
+
with sqlite3.connect(self.db_path) as conn:
|
| 73 |
+
rows = conn.execute(
|
| 74 |
+
"""
|
| 75 |
+
SELECT record_id, session_id, dataset_id, question, response_json, created_at
|
| 76 |
+
FROM session_records
|
| 77 |
+
WHERE session_id = ?
|
| 78 |
+
ORDER BY id ASC
|
| 79 |
+
""",
|
| 80 |
+
(session_id,),
|
| 81 |
+
).fetchall()
|
| 82 |
+
records: list[SessionRecord] = []
|
| 83 |
+
for record_id, sid, dataset_id, question, response_json, created_at in rows:
|
| 84 |
+
records.append(
|
| 85 |
+
SessionRecord(
|
| 86 |
+
record_id=record_id,
|
| 87 |
+
session_id=sid,
|
| 88 |
+
dataset_id=dataset_id,
|
| 89 |
+
question=question,
|
| 90 |
+
response=AnalysisResponse.model_validate(json.loads(response_json)),
|
| 91 |
+
created_at=created_at,
|
| 92 |
+
)
|
| 93 |
+
)
|
| 94 |
+
return records
|
| 95 |
+
|
| 96 |
+
def _init_db(self) -> None:
|
| 97 |
+
with sqlite3.connect(self.db_path) as conn:
|
| 98 |
+
conn.execute(
|
| 99 |
+
"""
|
| 100 |
+
CREATE TABLE IF NOT EXISTS session_records(
|
| 101 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 102 |
+
record_id TEXT NOT NULL UNIQUE,
|
| 103 |
+
session_id TEXT NOT NULL,
|
| 104 |
+
dataset_id TEXT NOT NULL,
|
| 105 |
+
question TEXT NOT NULL,
|
| 106 |
+
response_json TEXT NOT NULL,
|
| 107 |
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
| 108 |
+
)
|
| 109 |
+
"""
|
| 110 |
+
)
|
| 111 |
+
conn.execute(
|
| 112 |
+
"""
|
| 113 |
+
CREATE INDEX IF NOT EXISTS idx_session_records_session
|
| 114 |
+
ON session_records(session_id)
|
| 115 |
+
"""
|
| 116 |
+
)
|