Syntax Errors Fixed
Browse files- .gitignore +1 -1
- DEFAULT_AGENTS_SETUP.md +237 -0
- app.py +226 -138
- chat_database.db +2 -2
- entrypoint.sh +52 -21
- heatmap_correlation.png +3 -0
- load_default_agents.py +42 -0
- scripts/init_production_db.py +89 -34
- scripts/populate_agent_templates.py +5 -5
- src/agents/agents.py +557 -564
- src/agents/deep_agents.py +14 -8
- src/db/init_db.py +11 -9
- src/db/init_default_agents.py +281 -0
- src/managers/session_manager.py +12 -10
- src/managers/user_manager.py +50 -1
- src/routes/templates_routes.py +98 -35
.gitignore
CHANGED
|
@@ -25,7 +25,7 @@ migrations/
|
|
| 25 |
alembic.ini
|
| 26 |
|
| 27 |
|
| 28 |
-
*
|
| 29 |
|
| 30 |
schema*.md
|
| 31 |
|
|
|
|
| 25 |
alembic.ini
|
| 26 |
|
| 27 |
|
| 28 |
+
*.db
|
| 29 |
|
| 30 |
schema*.md
|
| 31 |
|
DEFAULT_AGENTS_SETUP.md
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Default Agents Setup Guide
|
| 2 |
+
|
| 3 |
+
This guide explains how to set up and use the default agents system in the Auto-Analyst backend.
|
| 4 |
+
|
| 5 |
+
## Overview
|
| 6 |
+
|
| 7 |
+
The system now includes 4 default agents that are stored in the database as templates:
|
| 8 |
+
|
| 9 |
+
1. **Data Preprocessing Agent** (`preprocessing_agent`) - Data cleaning and preparation
|
| 10 |
+
2. **Statistical Analytics Agent** (`statistical_analytics_agent`) - Statistical analysis using statsmodels
|
| 11 |
+
3. **Machine Learning Agent** (`sk_learn_agent`) - ML modeling using scikit-learn
|
| 12 |
+
4. **Data Visualization Agent** (`data_viz_agent`) - Interactive visualizations using Plotly
|
| 13 |
+
|
| 14 |
+
## Setup Instructions
|
| 15 |
+
|
| 16 |
+
### 1. Load Default Agents into Database
|
| 17 |
+
|
| 18 |
+
Run the setup script to populate the database with default agents:
|
| 19 |
+
|
| 20 |
+
```bash
|
| 21 |
+
cd Auto-Analyst-CS/auto-analyst-backend
|
| 22 |
+
python load_default_agents.py
|
| 23 |
+
```
|
| 24 |
+
|
| 25 |
+
**Or** use the API endpoint:
|
| 26 |
+
|
| 27 |
+
```bash
|
| 28 |
+
curl -X POST "http://localhost:8000/templates/load-default-agents" \
|
| 29 |
+
-H "Content-Type: application/json" \
|
| 30 |
+
-d '{"force_update": false}'
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
### 2. Agent Properties
|
| 34 |
+
|
| 35 |
+
All default agents are created with:
|
| 36 |
+
- `is_active = True` (available for use)
|
| 37 |
+
- `is_premium_only = False` (free to use)
|
| 38 |
+
- Proper categories (Data Manipulation, Statistical Analysis, Modelling, Visualization)
|
| 39 |
+
|
| 40 |
+
## User Preferences System
|
| 41 |
+
|
| 42 |
+
### Default Behavior
|
| 43 |
+
- **Default agents (preprocessing, statistical_analytics, sk_learn, data_viz) are ENABLED by default** for all users
|
| 44 |
+
- **Other templates are DISABLED by default** and must be explicitly enabled
|
| 45 |
+
- Templates can be used directly via `@template_name` regardless of preferences
|
| 46 |
+
|
| 47 |
+
### Managing User Preferences
|
| 48 |
+
|
| 49 |
+
#### Enable/Disable Templates
|
| 50 |
+
```bash
|
| 51 |
+
# Enable a template for a user
|
| 52 |
+
curl -X POST "http://localhost:8000/templates/user/1/template/1/toggle" \
|
| 53 |
+
-H "Content-Type: application/json" \
|
| 54 |
+
-d '{"is_enabled": true}'
|
| 55 |
+
|
| 56 |
+
# Disable a template for a user
|
| 57 |
+
curl -X POST "http://localhost:8000/templates/user/1/template/1/toggle" \
|
| 58 |
+
-H "Content-Type: application/json" \
|
| 59 |
+
-d '{"is_enabled": false}'
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
#### Bulk Enable/Disable
|
| 63 |
+
```bash
|
| 64 |
+
# Enable multiple templates at once
|
| 65 |
+
curl -X POST "http://localhost:8000/templates/user/1/bulk-toggle" \
|
| 66 |
+
-H "Content-Type: application/json" \
|
| 67 |
+
-d '{
|
| 68 |
+
"template_preferences": {
|
| 69 |
+
"1": true,
|
| 70 |
+
"2": true,
|
| 71 |
+
"3": false
|
| 72 |
+
}
|
| 73 |
+
}'
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
#### Get User's Template Preferences
|
| 77 |
+
```bash
|
| 78 |
+
# Get all templates with user's enabled/disabled status
|
| 79 |
+
curl "http://localhost:8000/templates/user/1"
|
| 80 |
+
|
| 81 |
+
# Get only enabled templates for user
|
| 82 |
+
curl "http://localhost:8000/templates/user/1/enabled"
|
| 83 |
+
|
| 84 |
+
# Get enabled templates for planner (max 10, ordered by usage)
|
| 85 |
+
curl "http://localhost:8000/templates/user/1/enabled/planner"
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
## Planner Integration
|
| 89 |
+
|
| 90 |
+
### How It Works
|
| 91 |
+
1. **Template Loading**: Only user-enabled templates are loaded into the planner
|
| 92 |
+
2. **No Agents Available**: If no templates are enabled, planner returns a helpful message
|
| 93 |
+
3. **Usage Tracking**: Template usage is tracked for prioritization
|
| 94 |
+
|
| 95 |
+
### Planner Response When No Agents Enabled
|
| 96 |
+
```json
|
| 97 |
+
{
|
| 98 |
+
"complexity": "no_agents_available",
|
| 99 |
+
"plan": "no_agents_available",
|
| 100 |
+
"plan_instructions": {
|
| 101 |
+
"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."
|
| 102 |
+
}
|
| 103 |
+
}
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
## API Endpoints
|
| 107 |
+
|
| 108 |
+
### Template Management
|
| 109 |
+
- `GET /templates/` - Get all available templates
|
| 110 |
+
- `GET /templates/template/{template_id}` - Get specific template
|
| 111 |
+
- `POST /templates/load-default-agents` - Load default agents into database
|
| 112 |
+
|
| 113 |
+
### User Preferences
|
| 114 |
+
- `GET /templates/user/{user_id}` - Get user's template preferences
|
| 115 |
+
- `GET /templates/user/{user_id}/enabled` - Get user's enabled templates
|
| 116 |
+
- `GET /templates/user/{user_id}/enabled/planner` - Get templates for planner (max 10)
|
| 117 |
+
- `POST /templates/user/{user_id}/template/{template_id}/toggle` - Toggle template preference
|
| 118 |
+
- `POST /templates/user/{user_id}/bulk-toggle` - Bulk toggle preferences
|
| 119 |
+
|
| 120 |
+
### Categories
|
| 121 |
+
- `GET /templates/categories/list` - Get all categories
|
| 122 |
+
- `GET /templates/categories` - Get templates grouped by category
|
| 123 |
+
- `GET /templates/category/{category}` - Get templates in specific category
|
| 124 |
+
|
| 125 |
+
## Usage Examples
|
| 126 |
+
|
| 127 |
+
### Frontend Integration
|
| 128 |
+
```typescript
|
| 129 |
+
// Enable preprocessing and visualization agents for user
|
| 130 |
+
const enableAgents = async (userId: number) => {
|
| 131 |
+
await fetch(`/templates/user/${userId}/bulk-toggle`, {
|
| 132 |
+
method: 'POST',
|
| 133 |
+
headers: { 'Content-Type': 'application/json' },
|
| 134 |
+
body: JSON.stringify({
|
| 135 |
+
template_preferences: {
|
| 136 |
+
"1": true, // preprocessing_agent
|
| 137 |
+
"4": true // data_viz_agent
|
| 138 |
+
}
|
| 139 |
+
})
|
| 140 |
+
});
|
| 141 |
+
};
|
| 142 |
+
|
| 143 |
+
// Get user's enabled templates
|
| 144 |
+
const getUserTemplates = async (userId: number) => {
|
| 145 |
+
const response = await fetch(`/templates/user/${userId}/enabled`);
|
| 146 |
+
return await response.json();
|
| 147 |
+
};
|
| 148 |
+
```
|
| 149 |
+
|
| 150 |
+
### Direct Agent Usage
|
| 151 |
+
Users can still use any agent directly regardless of preferences:
|
| 152 |
+
```
|
| 153 |
+
@preprocessing_agent clean this data
|
| 154 |
+
@data_viz_agent create a scatter plot of sales vs price
|
| 155 |
+
```
|
| 156 |
+
|
| 157 |
+
### Planner Usage
|
| 158 |
+
Only enabled agents will be available to the planner:
|
| 159 |
+
```
|
| 160 |
+
User: "Clean the data and create a visualization"
|
| 161 |
+
System: Uses only enabled agents to create the plan
|
| 162 |
+
```
|
| 163 |
+
|
| 164 |
+
## Database Schema
|
| 165 |
+
|
| 166 |
+
### AgentTemplate Table
|
| 167 |
+
```sql
|
| 168 |
+
CREATE TABLE agent_templates (
|
| 169 |
+
template_id SERIAL PRIMARY KEY,
|
| 170 |
+
template_name VARCHAR UNIQUE NOT NULL,
|
| 171 |
+
display_name VARCHAR,
|
| 172 |
+
description TEXT,
|
| 173 |
+
prompt_template TEXT,
|
| 174 |
+
category VARCHAR,
|
| 175 |
+
is_premium_only BOOLEAN DEFAULT FALSE,
|
| 176 |
+
is_active BOOLEAN DEFAULT TRUE,
|
| 177 |
+
created_at TIMESTAMP,
|
| 178 |
+
updated_at TIMESTAMP
|
| 179 |
+
);
|
| 180 |
+
```
|
| 181 |
+
|
| 182 |
+
### UserTemplatePreference Table
|
| 183 |
+
```sql
|
| 184 |
+
CREATE TABLE user_template_preferences (
|
| 185 |
+
user_id INTEGER,
|
| 186 |
+
template_id INTEGER,
|
| 187 |
+
is_enabled BOOLEAN DEFAULT FALSE,
|
| 188 |
+
usage_count INTEGER DEFAULT 0,
|
| 189 |
+
last_used_at TIMESTAMP,
|
| 190 |
+
created_at TIMESTAMP,
|
| 191 |
+
updated_at TIMESTAMP,
|
| 192 |
+
PRIMARY KEY (user_id, template_id),
|
| 193 |
+
FOREIGN KEY (user_id) REFERENCES users(user_id),
|
| 194 |
+
FOREIGN KEY (template_id) REFERENCES agent_templates(template_id)
|
| 195 |
+
);
|
| 196 |
+
```
|
| 197 |
+
|
| 198 |
+
## Troubleshooting
|
| 199 |
+
|
| 200 |
+
### Common Issues
|
| 201 |
+
|
| 202 |
+
1. **No agents available in planner**
|
| 203 |
+
- Check if user has enabled any templates: `GET /templates/user/{user_id}/enabled`
|
| 204 |
+
- Enable templates using the toggle endpoint
|
| 205 |
+
|
| 206 |
+
2. **Default agents not found**
|
| 207 |
+
- Run the load script: `python load_default_agents.py`
|
| 208 |
+
- Check if agents exist: `GET /templates/`
|
| 209 |
+
|
| 210 |
+
3. **Import errors in load script**
|
| 211 |
+
- Ensure you're in the backend directory
|
| 212 |
+
- Check that all dependencies are installed
|
| 213 |
+
- Verify database connection
|
| 214 |
+
|
| 215 |
+
### Logs
|
| 216 |
+
Check the application logs for detailed error messages:
|
| 217 |
+
```bash
|
| 218 |
+
tail -f logs/templates_routes.log
|
| 219 |
+
tail -f logs/agents.log
|
| 220 |
+
```
|
| 221 |
+
|
| 222 |
+
## Migration from Old System
|
| 223 |
+
|
| 224 |
+
If migrating from the previous custom agents system:
|
| 225 |
+
|
| 226 |
+
1. **Data Migration**: Existing custom agents should be migrated to the new template system
|
| 227 |
+
2. **User Preferences**: Users will need to re-enable their preferred agents
|
| 228 |
+
3. **API Updates**: Update frontend code to use new template endpoints
|
| 229 |
+
4. **Testing**: Verify planner works with enabled templates only
|
| 230 |
+
|
| 231 |
+
## Support
|
| 232 |
+
|
| 233 |
+
For issues or questions:
|
| 234 |
+
1. Check the logs for error messages
|
| 235 |
+
2. Verify database connections
|
| 236 |
+
3. Ensure proper API endpoint usage
|
| 237 |
+
4. Test with the load script first
|
app.py
CHANGED
|
@@ -255,19 +255,7 @@ if not os.path.exists(housing_csv_path):
|
|
| 255 |
logger.log_message(f"Housing.csv not found at {os.path.abspath(housing_csv_path)}", level=logging.ERROR)
|
| 256 |
raise FileNotFoundError(f"Housing.csv not found at {os.path.abspath(housing_csv_path)}")
|
| 257 |
|
| 258 |
-
|
| 259 |
-
"data_viz_agent": data_viz_agent,
|
| 260 |
-
"sk_learn_agent": sk_learn_agent,
|
| 261 |
-
"statistical_analytics_agent": statistical_analytics_agent,
|
| 262 |
-
"preprocessing_agent": preprocessing_agent,
|
| 263 |
-
}
|
| 264 |
-
|
| 265 |
-
PLANNER_AGENTS = {
|
| 266 |
-
"planner_preprocessing_agent": planner_preprocessing_agent,
|
| 267 |
-
"planner_sk_learn_agent": planner_sk_learn_agent,
|
| 268 |
-
"planner_statistical_analytics_agent": planner_statistical_analytics_agent,
|
| 269 |
-
"planner_data_viz_agent": planner_data_viz_agent,
|
| 270 |
-
}
|
| 271 |
|
| 272 |
# Add session header
|
| 273 |
X_SESSION_ID = APIKeyHeader(name="X-Session-ID", auto_error=False)
|
|
@@ -275,7 +263,7 @@ X_SESSION_ID = APIKeyHeader(name="X-Session-ID", auto_error=False)
|
|
| 275 |
# Update AppState class to use SessionManager
|
| 276 |
class AppState:
|
| 277 |
def __init__(self):
|
| 278 |
-
self._session_manager = SessionManager(styling_instructions,
|
| 279 |
self.model_config = DEFAULT_MODEL_CONFIG.copy()
|
| 280 |
# Update the SessionManager with the current model_config
|
| 281 |
self._session_manager._app_model_config = self.model_config
|
|
@@ -326,17 +314,82 @@ class AppState:
|
|
| 326 |
def get_deep_analyzer(self, session_id: str):
|
| 327 |
"""Get or create deep analysis module for a session"""
|
| 328 |
session_state = self.get_session_state(session_id)
|
| 329 |
-
|
| 330 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
deep_agents = {
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
|
| 338 |
-
deep_agents_desc = PLANNER_AGENTS_WITH_DESCRIPTION
|
| 339 |
session_state['deep_analyzer'] = deep_analysis_module(agents=deep_agents, agents_desc=deep_agents_desc)
|
|
|
|
|
|
|
|
|
|
| 340 |
|
| 341 |
return session_state['deep_analyzer']
|
| 342 |
|
|
@@ -344,6 +397,7 @@ class AppState:
|
|
| 344 |
app = FastAPI(title="AI Analytics API", version="1.0")
|
| 345 |
app.state = AppState()
|
| 346 |
|
|
|
|
| 347 |
# Configure middleware
|
| 348 |
# Use a wildcard for local development or read from environment
|
| 349 |
is_development = os.getenv("ENVIRONMENT", "development").lower() == "development"
|
|
@@ -450,15 +504,15 @@ async def chat_with_agent(
|
|
| 450 |
timeout=REQUEST_TIMEOUT_SECONDS
|
| 451 |
)
|
| 452 |
else:
|
| 453 |
-
# All standard/template agents - use auto_analyst_ind
|
| 454 |
-
standard_agent_sigs = [AVAILABLE_AGENTS[agent] for agent in standard_agents]
|
| 455 |
user_id = session_state.get("user_id")
|
| 456 |
|
| 457 |
-
# Create database session for
|
| 458 |
from src.db.init_db import session_factory
|
| 459 |
db_session = session_factory()
|
| 460 |
try:
|
| 461 |
-
|
|
|
|
| 462 |
session_lm = get_session_lm(session_state)
|
| 463 |
with dspy.context(lm=session_lm):
|
| 464 |
response = await asyncio.wait_for(
|
|
@@ -469,31 +523,15 @@ async def chat_with_agent(
|
|
| 469 |
db_session.close()
|
| 470 |
else:
|
| 471 |
# Single agent case
|
| 472 |
-
if _is_standard_agent(agent_name):
|
| 473 |
-
# Standard agent - use auto_analyst_ind
|
| 474 |
-
user_id = session_state.get("user_id")
|
| 475 |
-
|
| 476 |
-
# Create database session for template loading
|
| 477 |
-
from src.db.init_db import session_factory
|
| 478 |
-
db_session = session_factory()
|
| 479 |
-
try:
|
| 480 |
-
agent = auto_analyst_ind(agents=[AVAILABLE_AGENTS[agent_name]], retrievers=session_state["retrievers"], user_id=user_id, db_session=db_session)
|
| 481 |
-
session_lm = get_session_lm(session_state)
|
| 482 |
-
with dspy.context(lm=session_lm):
|
| 483 |
-
response = await asyncio.wait_for(
|
| 484 |
-
agent.forward(enhanced_query, agent_name),
|
| 485 |
-
timeout=REQUEST_TIMEOUT_SECONDS
|
| 486 |
-
)
|
| 487 |
-
finally:
|
| 488 |
-
db_session.close()
|
| 489 |
-
elif _is_template_agent(agent_name):
|
| 490 |
-
# Template agent - use auto_analyst_ind with empty agents list (templates loaded in init)
|
| 491 |
user_id = session_state.get("user_id")
|
| 492 |
|
| 493 |
-
# Create database session for
|
| 494 |
from src.db.init_db import session_factory
|
| 495 |
db_session = session_factory()
|
| 496 |
try:
|
|
|
|
| 497 |
agent = auto_analyst_ind(agents=[], retrievers=session_state["retrievers"], user_id=user_id, db_session=db_session)
|
| 498 |
session_lm = get_session_lm(session_state)
|
| 499 |
with dspy.context(lm=session_lm):
|
|
@@ -614,48 +652,37 @@ def _update_session_from_query_params(request_obj: Request, session_state: dict)
|
|
| 614 |
|
| 615 |
|
| 616 |
def _validate_agent_name(agent_name: str, session_state: dict = None):
|
| 617 |
-
"""Validate that the
|
| 618 |
if "," in agent_name:
|
|
|
|
| 619 |
agent_list = [agent.strip() for agent in agent_name.split(",")]
|
| 620 |
for agent in agent_list:
|
| 621 |
if not _is_agent_available(agent, session_state):
|
| 622 |
available_agents = _get_available_agents_list(session_state)
|
| 623 |
raise HTTPException(
|
| 624 |
-
status_code=
|
| 625 |
detail=f"Agent '{agent}' not found. Available agents: {available_agents}"
|
| 626 |
)
|
| 627 |
-
|
| 628 |
-
|
|
|
|
|
|
|
| 629 |
raise HTTPException(
|
| 630 |
-
|
| 631 |
-
|
| 632 |
)
|
| 633 |
|
| 634 |
def _is_agent_available(agent_name: str, session_state: dict = None) -> bool:
|
| 635 |
-
"""Check if agent is available
|
| 636 |
-
# Check standard
|
| 637 |
-
if agent_name
|
| 638 |
return True
|
| 639 |
|
| 640 |
-
# Check template
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
from src.db.schemas.models import AgentTemplate
|
| 644 |
-
|
| 645 |
-
db_session = session_factory()
|
| 646 |
-
try:
|
| 647 |
-
template = db_session.query(AgentTemplate).filter(
|
| 648 |
-
AgentTemplate.template_name == agent_name,
|
| 649 |
-
AgentTemplate.is_active == True
|
| 650 |
-
).first()
|
| 651 |
-
if template:
|
| 652 |
-
return True
|
| 653 |
-
finally:
|
| 654 |
-
db_session.close()
|
| 655 |
-
except Exception as e:
|
| 656 |
-
logger.log_message(f"Error checking template availability for {agent_name}: {str(e)}", level=logging.ERROR)
|
| 657 |
|
| 658 |
-
# Check
|
| 659 |
if session_state and "ai_system" in session_state:
|
| 660 |
ai_system = session_state["ai_system"]
|
| 661 |
if hasattr(ai_system, 'agents') and agent_name in ai_system.agents:
|
|
@@ -664,22 +691,32 @@ def _is_agent_available(agent_name: str, session_state: dict = None) -> bool:
|
|
| 664 |
return False
|
| 665 |
|
| 666 |
def _get_available_agents_list(session_state: dict = None) -> list:
|
| 667 |
-
"""Get list of all available agents
|
| 668 |
-
|
|
|
|
| 669 |
|
| 670 |
-
#
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 677 |
|
| 678 |
return available
|
| 679 |
|
| 680 |
def _is_standard_agent(agent_name: str) -> bool:
|
| 681 |
-
"""Check if agent is
|
| 682 |
-
|
|
|
|
| 683 |
|
| 684 |
def _is_template_agent(agent_name: str) -> bool:
|
| 685 |
"""Check if agent is a template agent"""
|
|
@@ -999,66 +1036,54 @@ async def _execute_plan_with_timeout(ai_system, enhanced_query, plan_response):
|
|
| 999 |
# Add an endpoint to list available agents
|
| 1000 |
@app.get("/agents", response_model=dict)
|
| 1001 |
async def list_agents(request: Request, session_id: str = Depends(get_session_id_dependency)):
|
|
|
|
| 1002 |
session_state = app.state.get_session_state(session_id)
|
| 1003 |
|
| 1004 |
-
# Check if user_id is provided in query params to associate with session
|
| 1005 |
-
user_id_param = request.query_params.get("user_id")
|
| 1006 |
-
if user_id_param:
|
| 1007 |
-
try:
|
| 1008 |
-
user_id = int(user_id_param)
|
| 1009 |
-
# Associate the user with this session to load custom agents
|
| 1010 |
-
app.state.set_session_user(session_id, user_id)
|
| 1011 |
-
# Refresh session state after user association
|
| 1012 |
-
session_state = app.state.get_session_state(session_id)
|
| 1013 |
-
except (ValueError, TypeError):
|
| 1014 |
-
logger.log_message(f"Invalid user_id in agents endpoint: {user_id_param}", level=logging.WARNING)
|
| 1015 |
-
|
| 1016 |
-
# Get user-specific agent list including custom agents
|
| 1017 |
-
available_agents_list = _get_available_agents_list(session_state)
|
| 1018 |
-
standard_agents = list(AVAILABLE_AGENTS.keys())
|
| 1019 |
-
planner_agents = list(PLANNER_AGENTS.keys())
|
| 1020 |
-
|
| 1021 |
-
# Get template agents from database
|
| 1022 |
-
template_agents = []
|
| 1023 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1024 |
from src.db.init_db import session_factory
|
| 1025 |
-
from src.
|
| 1026 |
|
| 1027 |
db_session = session_factory()
|
| 1028 |
try:
|
| 1029 |
-
|
| 1030 |
-
|
| 1031 |
-
|
| 1032 |
-
|
| 1033 |
-
|
| 1034 |
-
logger.log_message(f"
|
| 1035 |
-
|
| 1036 |
finally:
|
| 1037 |
db_session.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1038 |
except Exception as e:
|
| 1039 |
-
logger.log_message(f"Error
|
| 1040 |
-
|
| 1041 |
-
# Custom agents are user-created agents (not standard, not planner, not template)
|
| 1042 |
-
custom_agents = [agent for agent in available_agents_list
|
| 1043 |
-
if agent not in standard_agents and agent not in planner_agents and agent not in template_agents]
|
| 1044 |
-
|
| 1045 |
-
# Add template agents to available agents list if they're not already there
|
| 1046 |
-
for template_agent in template_agents:
|
| 1047 |
-
if template_agent not in available_agents_list:
|
| 1048 |
-
available_agents_list.append(template_agent)
|
| 1049 |
-
|
| 1050 |
-
return {
|
| 1051 |
-
"available_agents": available_agents_list,
|
| 1052 |
-
"standard_agents": standard_agents,
|
| 1053 |
-
"custom_agents": custom_agents,
|
| 1054 |
-
"template_agents": template_agents,
|
| 1055 |
-
"planner_agents": planner_agents,
|
| 1056 |
-
"deep_analysis": {
|
| 1057 |
-
"available": True,
|
| 1058 |
-
"description": "Comprehensive multi-step analysis with automated planning"
|
| 1059 |
-
},
|
| 1060 |
-
"description": "List of available specialized agents that can be called using @agent_name"
|
| 1061 |
-
}
|
| 1062 |
|
| 1063 |
@app.get("/health", response_model=dict)
|
| 1064 |
async def health():
|
|
@@ -1162,7 +1187,7 @@ async def deep_analysis_streaming(
|
|
| 1162 |
session_lm = dspy.LM(model="anthropic/claude-sonnet-4-20250514", max_tokens=7000, temperature=0.5)
|
| 1163 |
|
| 1164 |
return StreamingResponse(
|
| 1165 |
-
_generate_deep_analysis_stream(session_state, request.goal, session_lm),
|
| 1166 |
media_type='text/event-stream',
|
| 1167 |
headers={
|
| 1168 |
'Cache-Control': 'no-cache',
|
|
@@ -1179,7 +1204,7 @@ async def deep_analysis_streaming(
|
|
| 1179 |
logger.log_message(f"Streaming deep analysis failed: {str(e)}", level=logging.ERROR)
|
| 1180 |
raise HTTPException(status_code=500, detail=f"Streaming deep analysis failed: {str(e)}")
|
| 1181 |
|
| 1182 |
-
async def _generate_deep_analysis_stream(session_state: dict, goal: str, session_lm):
|
| 1183 |
"""Generate streaming responses for deep analysis"""
|
| 1184 |
# Track the start time for duration calculation
|
| 1185 |
start_time = datetime.now(UTC)
|
|
@@ -1285,8 +1310,9 @@ async def _generate_deep_analysis_stream(session_state: dict, goal: str, session
|
|
| 1285 |
# Update DB status to running
|
| 1286 |
await update_report_in_db("running", 5)
|
| 1287 |
|
| 1288 |
-
# Get deep analyzer
|
| 1289 |
-
|
|
|
|
| 1290 |
|
| 1291 |
# Make the dataset available globally for code execution
|
| 1292 |
globals()['df'] = df
|
|
@@ -1510,6 +1536,68 @@ async def download_html_report(
|
|
| 1510 |
logger.log_message(f"Failed to generate HTML report: {str(e)}", level=logging.ERROR)
|
| 1511 |
raise HTTPException(status_code=500, detail=f"Failed to generate report: {str(e)}")
|
| 1512 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1513 |
# In the section where routers are included, add the session_router
|
| 1514 |
app.include_router(chat_router)
|
| 1515 |
app.include_router(analytics_router)
|
|
|
|
| 255 |
logger.log_message(f"Housing.csv not found at {os.path.abspath(housing_csv_path)}", level=logging.ERROR)
|
| 256 |
raise FileNotFoundError(f"Housing.csv not found at {os.path.abspath(housing_csv_path)}")
|
| 257 |
|
| 258 |
+
# All agents are now loaded from database - no hardcoded dictionaries needed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
|
| 260 |
# Add session header
|
| 261 |
X_SESSION_ID = APIKeyHeader(name="X-Session-ID", auto_error=False)
|
|
|
|
| 263 |
# Update AppState class to use SessionManager
|
| 264 |
class AppState:
|
| 265 |
def __init__(self):
|
| 266 |
+
self._session_manager = SessionManager(styling_instructions, {}) # Empty dict, agents loaded from DB
|
| 267 |
self.model_config = DEFAULT_MODEL_CONFIG.copy()
|
| 268 |
# Update the SessionManager with the current model_config
|
| 269 |
self._session_manager._app_model_config = self.model_config
|
|
|
|
| 314 |
def get_deep_analyzer(self, session_id: str):
|
| 315 |
"""Get or create deep analysis module for a session"""
|
| 316 |
session_state = self.get_session_state(session_id)
|
| 317 |
+
user_id = session_state.get("user_id")
|
| 318 |
+
|
| 319 |
+
# Check if we need to recreate the deep analyzer (user changed or doesn't exist)
|
| 320 |
+
current_analyzer = session_state.get('deep_analyzer')
|
| 321 |
+
analyzer_user_id = session_state.get('deep_analyzer_user_id')
|
| 322 |
+
|
| 323 |
+
logger.log_message(f"Deep analyzer check - session: {session_id}, current_user: {user_id}, analyzer_user: {analyzer_user_id}, has_analyzer: {current_analyzer is not None}", level=logging.INFO)
|
| 324 |
+
|
| 325 |
+
if (not current_analyzer or
|
| 326 |
+
analyzer_user_id != user_id or
|
| 327 |
+
not hasattr(session_state, 'deep_analyzer')):
|
| 328 |
+
|
| 329 |
+
logger.log_message(f"Creating/recreating deep analyzer for session {session_id}, user_id: {user_id} (reason: analyzer_exists={current_analyzer is not None}, user_match={analyzer_user_id == user_id})", level=logging.INFO)
|
| 330 |
+
|
| 331 |
+
# Load user-enabled agents from database using preference system
|
| 332 |
+
from src.db.init_db import session_factory
|
| 333 |
+
from src.agents.agents import load_user_enabled_templates_for_planner_from_db
|
| 334 |
+
|
| 335 |
+
db_session = session_factory()
|
| 336 |
+
try:
|
| 337 |
+
# Load user-enabled agents for planner (respects preferences)
|
| 338 |
+
if user_id:
|
| 339 |
+
enabled_agents_dict = load_user_enabled_templates_for_planner_from_db(user_id, db_session)
|
| 340 |
+
logger.log_message(f"Deep analyzer loaded {len(enabled_agents_dict)} enabled agents for user {user_id}: {list(enabled_agents_dict.keys())}", level=logging.INFO)
|
| 341 |
+
|
| 342 |
+
if not enabled_agents_dict:
|
| 343 |
+
logger.log_message(f"WARNING: No enabled agents found for user {user_id}, falling back to defaults", level=logging.WARNING)
|
| 344 |
+
# Fallback to default agents if no enabled agents
|
| 345 |
+
from src.agents.agents import preprocessing_agent, statistical_analytics_agent, sk_learn_agent, data_viz_agent
|
| 346 |
+
enabled_agents_dict = {
|
| 347 |
+
"preprocessing_agent": preprocessing_agent,
|
| 348 |
+
"statistical_analytics_agent": statistical_analytics_agent,
|
| 349 |
+
"sk_learn_agent": sk_learn_agent,
|
| 350 |
+
"data_viz_agent": data_viz_agent
|
| 351 |
+
}
|
| 352 |
+
else:
|
| 353 |
+
# Fallback to default agents if no user_id
|
| 354 |
+
logger.log_message("No user_id in session, loading default agents for deep analysis", level=logging.WARNING)
|
| 355 |
+
from src.agents.agents import preprocessing_agent, statistical_analytics_agent, sk_learn_agent, data_viz_agent
|
| 356 |
+
enabled_agents_dict = {
|
| 357 |
+
"preprocessing_agent": preprocessing_agent,
|
| 358 |
+
"statistical_analytics_agent": statistical_analytics_agent,
|
| 359 |
+
"sk_learn_agent": sk_learn_agent,
|
| 360 |
+
"data_viz_agent": data_viz_agent
|
| 361 |
+
}
|
| 362 |
+
|
| 363 |
+
# Create agents dictionary for deep analysis using enabled agents
|
| 364 |
+
deep_agents = {}
|
| 365 |
+
deep_agents_desc = {}
|
| 366 |
+
|
| 367 |
+
for agent_name, signature in enabled_agents_dict.items():
|
| 368 |
+
deep_agents[agent_name] = dspy.asyncify(dspy.ChainOfThought(signature))
|
| 369 |
+
# Get agent description from database
|
| 370 |
+
deep_agents_desc[agent_name] = get_agent_description(agent_name)
|
| 371 |
+
|
| 372 |
+
logger.log_message(f"Deep analyzer initialized with {len(deep_agents)} agents: {list(deep_agents.keys())}", level=logging.INFO)
|
| 373 |
+
|
| 374 |
+
except Exception as e:
|
| 375 |
+
logger.log_message(f"Error loading agents for deep analysis: {str(e)}", level=logging.ERROR)
|
| 376 |
+
# Fallback to minimal set
|
| 377 |
+
from src.agents.agents import preprocessing_agent, statistical_analytics_agent, sk_learn_agent, data_viz_agent
|
| 378 |
deep_agents = {
|
| 379 |
+
"preprocessing_agent": dspy.asyncify(dspy.ChainOfThought(preprocessing_agent)),
|
| 380 |
+
"statistical_analytics_agent": dspy.asyncify(dspy.ChainOfThought(statistical_analytics_agent)),
|
| 381 |
+
"sk_learn_agent": dspy.asyncify(dspy.ChainOfThought(sk_learn_agent)),
|
| 382 |
+
"data_viz_agent": dspy.asyncify(dspy.ChainOfThought(data_viz_agent))
|
| 383 |
+
}
|
| 384 |
+
deep_agents_desc = {name: get_agent_description(name) for name in deep_agents.keys()}
|
| 385 |
+
logger.log_message(f"Using fallback agents: {list(deep_agents.keys())}", level=logging.WARNING)
|
| 386 |
+
finally:
|
| 387 |
+
db_session.close()
|
| 388 |
|
|
|
|
| 389 |
session_state['deep_analyzer'] = deep_analysis_module(agents=deep_agents, agents_desc=deep_agents_desc)
|
| 390 |
+
session_state['deep_analyzer_user_id'] = user_id # Track which user this analyzer was created for
|
| 391 |
+
else:
|
| 392 |
+
logger.log_message(f"Using existing deep analyzer for session {session_id}, user_id: {user_id}", level=logging.INFO)
|
| 393 |
|
| 394 |
return session_state['deep_analyzer']
|
| 395 |
|
|
|
|
| 397 |
app = FastAPI(title="AI Analytics API", version="1.0")
|
| 398 |
app.state = AppState()
|
| 399 |
|
| 400 |
+
|
| 401 |
# Configure middleware
|
| 402 |
# Use a wildcard for local development or read from environment
|
| 403 |
is_development = os.getenv("ENVIRONMENT", "development").lower() == "development"
|
|
|
|
| 504 |
timeout=REQUEST_TIMEOUT_SECONDS
|
| 505 |
)
|
| 506 |
else:
|
| 507 |
+
# All standard/template agents - use auto_analyst_ind which loads from DB
|
|
|
|
| 508 |
user_id = session_state.get("user_id")
|
| 509 |
|
| 510 |
+
# Create database session for agent loading
|
| 511 |
from src.db.init_db import session_factory
|
| 512 |
db_session = session_factory()
|
| 513 |
try:
|
| 514 |
+
# auto_analyst_ind will load all agents from database
|
| 515 |
+
agent = auto_analyst_ind(agents=[], retrievers=session_state["retrievers"], user_id=user_id, db_session=db_session)
|
| 516 |
session_lm = get_session_lm(session_state)
|
| 517 |
with dspy.context(lm=session_lm):
|
| 518 |
response = await asyncio.wait_for(
|
|
|
|
| 523 |
db_session.close()
|
| 524 |
else:
|
| 525 |
# Single agent case
|
| 526 |
+
if _is_standard_agent(agent_name) or _is_template_agent(agent_name):
|
| 527 |
+
# Standard or template agent - use auto_analyst_ind which loads from DB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 528 |
user_id = session_state.get("user_id")
|
| 529 |
|
| 530 |
+
# Create database session for agent loading
|
| 531 |
from src.db.init_db import session_factory
|
| 532 |
db_session = session_factory()
|
| 533 |
try:
|
| 534 |
+
# auto_analyst_ind will load all agents from database
|
| 535 |
agent = auto_analyst_ind(agents=[], retrievers=session_state["retrievers"], user_id=user_id, db_session=db_session)
|
| 536 |
session_lm = get_session_lm(session_state)
|
| 537 |
with dspy.context(lm=session_lm):
|
|
|
|
| 652 |
|
| 653 |
|
| 654 |
def _validate_agent_name(agent_name: str, session_state: dict = None):
|
| 655 |
+
"""Validate that the agent name(s) are available"""
|
| 656 |
if "," in agent_name:
|
| 657 |
+
# Multiple agents
|
| 658 |
agent_list = [agent.strip() for agent in agent_name.split(",")]
|
| 659 |
for agent in agent_list:
|
| 660 |
if not _is_agent_available(agent, session_state):
|
| 661 |
available_agents = _get_available_agents_list(session_state)
|
| 662 |
raise HTTPException(
|
| 663 |
+
status_code=400,
|
| 664 |
detail=f"Agent '{agent}' not found. Available agents: {available_agents}"
|
| 665 |
)
|
| 666 |
+
else:
|
| 667 |
+
# Single agent
|
| 668 |
+
if not _is_agent_available(agent_name, session_state):
|
| 669 |
+
available_agents = _get_available_agents_list(session_state)
|
| 670 |
raise HTTPException(
|
| 671 |
+
status_code=400,
|
| 672 |
+
detail=f"Agent '{agent_name}' not found. Available agents: {available_agents}"
|
| 673 |
)
|
| 674 |
|
| 675 |
def _is_agent_available(agent_name: str, session_state: dict = None) -> bool:
|
| 676 |
+
"""Check if an agent is available (standard, template, or custom)"""
|
| 677 |
+
# Check if it's a standard agent
|
| 678 |
+
if _is_standard_agent(agent_name):
|
| 679 |
return True
|
| 680 |
|
| 681 |
+
# Check if it's a template agent
|
| 682 |
+
if _is_template_agent(agent_name):
|
| 683 |
+
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 684 |
|
| 685 |
+
# Check if it's a custom agent in session
|
| 686 |
if session_state and "ai_system" in session_state:
|
| 687 |
ai_system = session_state["ai_system"]
|
| 688 |
if hasattr(ai_system, 'agents') and agent_name in ai_system.agents:
|
|
|
|
| 691 |
return False
|
| 692 |
|
| 693 |
def _get_available_agents_list(session_state: dict = None) -> list:
|
| 694 |
+
"""Get list of all available agents from database"""
|
| 695 |
+
from src.db.init_db import session_factory
|
| 696 |
+
from src.agents.agents import load_all_available_templates_from_db
|
| 697 |
|
| 698 |
+
# Core agents (always available)
|
| 699 |
+
available = ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"]
|
| 700 |
+
|
| 701 |
+
# Add template agents from database
|
| 702 |
+
db_session = session_factory()
|
| 703 |
+
try:
|
| 704 |
+
template_agents_dict = load_all_available_templates_from_db(db_session)
|
| 705 |
+
# template_agents_dict is a dict with template_name as keys
|
| 706 |
+
template_names = [template_name for template_name in template_agents_dict.keys()
|
| 707 |
+
if template_name not in available and template_name != 'basic_qa_agent']
|
| 708 |
+
available.extend(template_names)
|
| 709 |
+
except Exception as e:
|
| 710 |
+
logger.log_message(f"Error loading template agents: {str(e)}", level=logging.ERROR)
|
| 711 |
+
finally:
|
| 712 |
+
db_session.close()
|
| 713 |
|
| 714 |
return available
|
| 715 |
|
| 716 |
def _is_standard_agent(agent_name: str) -> bool:
|
| 717 |
+
"""Check if agent is one of the 4 core standard agents"""
|
| 718 |
+
standard_agents = ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"]
|
| 719 |
+
return agent_name in standard_agents
|
| 720 |
|
| 721 |
def _is_template_agent(agent_name: str) -> bool:
|
| 722 |
"""Check if agent is a template agent"""
|
|
|
|
| 1036 |
# Add an endpoint to list available agents
|
| 1037 |
@app.get("/agents", response_model=dict)
|
| 1038 |
async def list_agents(request: Request, session_id: str = Depends(get_session_id_dependency)):
|
| 1039 |
+
"""Get all available agents (standard, template, and custom)"""
|
| 1040 |
session_state = app.state.get_session_state(session_id)
|
| 1041 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1042 |
try:
|
| 1043 |
+
# Get all available agents from database and session
|
| 1044 |
+
available_agents_list = _get_available_agents_list(session_state)
|
| 1045 |
+
|
| 1046 |
+
# Categorize agents
|
| 1047 |
+
standard_agents = ["preprocessing_agent", "statistical_analytics_agent", "sk_learn_agent", "data_viz_agent"]
|
| 1048 |
+
|
| 1049 |
+
# Get template agents from database
|
| 1050 |
from src.db.init_db import session_factory
|
| 1051 |
+
from src.agents.agents import load_all_available_templates_from_db
|
| 1052 |
|
| 1053 |
db_session = session_factory()
|
| 1054 |
try:
|
| 1055 |
+
template_agents_dict = load_all_available_templates_from_db(db_session)
|
| 1056 |
+
# template_agents_dict is a dict with template_name as keys
|
| 1057 |
+
template_agents = [template_name for template_name in template_agents_dict.keys()
|
| 1058 |
+
if template_name not in standard_agents and template_name != 'basic_qa_agent']
|
| 1059 |
+
except Exception as e:
|
| 1060 |
+
logger.log_message(f"Error loading template agents in /agents endpoint: {str(e)}", level=logging.ERROR)
|
| 1061 |
+
template_agents = []
|
| 1062 |
finally:
|
| 1063 |
db_session.close()
|
| 1064 |
+
|
| 1065 |
+
# Get custom agents from session
|
| 1066 |
+
custom_agents = []
|
| 1067 |
+
if session_state and "ai_system" in session_state:
|
| 1068 |
+
ai_system = session_state["ai_system"]
|
| 1069 |
+
if hasattr(ai_system, 'agents'):
|
| 1070 |
+
custom_agents = [agent for agent in available_agents_list
|
| 1071 |
+
if agent not in standard_agents and agent not in template_agents]
|
| 1072 |
+
|
| 1073 |
+
# Ensure template agents are in the available list
|
| 1074 |
+
for template_agent in template_agents:
|
| 1075 |
+
if template_agent not in available_agents_list:
|
| 1076 |
+
available_agents_list.append(template_agent)
|
| 1077 |
+
|
| 1078 |
+
return {
|
| 1079 |
+
"available_agents": available_agents_list,
|
| 1080 |
+
"standard_agents": standard_agents,
|
| 1081 |
+
"template_agents": template_agents,
|
| 1082 |
+
"custom_agents": custom_agents
|
| 1083 |
+
}
|
| 1084 |
except Exception as e:
|
| 1085 |
+
logger.log_message(f"Error getting agents list: {str(e)}", level=logging.ERROR)
|
| 1086 |
+
raise HTTPException(status_code=500, detail=f"Error getting agents list: {str(e)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1087 |
|
| 1088 |
@app.get("/health", response_model=dict)
|
| 1089 |
async def health():
|
|
|
|
| 1187 |
session_lm = dspy.LM(model="anthropic/claude-sonnet-4-20250514", max_tokens=7000, temperature=0.5)
|
| 1188 |
|
| 1189 |
return StreamingResponse(
|
| 1190 |
+
_generate_deep_analysis_stream(session_state, request.goal, session_lm, session_id),
|
| 1191 |
media_type='text/event-stream',
|
| 1192 |
headers={
|
| 1193 |
'Cache-Control': 'no-cache',
|
|
|
|
| 1204 |
logger.log_message(f"Streaming deep analysis failed: {str(e)}", level=logging.ERROR)
|
| 1205 |
raise HTTPException(status_code=500, detail=f"Streaming deep analysis failed: {str(e)}")
|
| 1206 |
|
| 1207 |
+
async def _generate_deep_analysis_stream(session_state: dict, goal: str, session_lm, session_id: str):
|
| 1208 |
"""Generate streaming responses for deep analysis"""
|
| 1209 |
# Track the start time for duration calculation
|
| 1210 |
start_time = datetime.now(UTC)
|
|
|
|
| 1310 |
# Update DB status to running
|
| 1311 |
await update_report_in_db("running", 5)
|
| 1312 |
|
| 1313 |
+
# Get deep analyzer - use the correct session_id from the session_state
|
| 1314 |
+
logger.log_message(f"Getting deep analyzer for session_id: {session_id}, user_id: {user_id}", level=logging.INFO)
|
| 1315 |
+
deep_analyzer = app.state.get_deep_analyzer(session_id)
|
| 1316 |
|
| 1317 |
# Make the dataset available globally for code execution
|
| 1318 |
globals()['df'] = df
|
|
|
|
| 1536 |
logger.log_message(f"Failed to generate HTML report: {str(e)}", level=logging.ERROR)
|
| 1537 |
raise HTTPException(status_code=500, detail=f"Failed to generate report: {str(e)}")
|
| 1538 |
|
| 1539 |
+
@app.get("/debug/deep_analysis_agents")
|
| 1540 |
+
async def debug_deep_analysis_agents(session_id: str = Depends(get_session_id_dependency)):
|
| 1541 |
+
"""Debug endpoint to show which agents are loaded for deep analysis"""
|
| 1542 |
+
session_state = app.state.get_session_state(session_id)
|
| 1543 |
+
user_id = session_state.get("user_id")
|
| 1544 |
+
|
| 1545 |
+
try:
|
| 1546 |
+
# Get the deep analyzer for this session
|
| 1547 |
+
deep_analyzer = app.state.get_deep_analyzer(session_id)
|
| 1548 |
+
|
| 1549 |
+
# Get the agents from the deep analyzer
|
| 1550 |
+
available_agents = list(deep_analyzer.agents.keys()) if hasattr(deep_analyzer, 'agents') else []
|
| 1551 |
+
|
| 1552 |
+
# Also get the raw enabled agents from database
|
| 1553 |
+
from src.db.init_db import session_factory
|
| 1554 |
+
from src.agents.agents import load_user_enabled_templates_for_planner_from_db
|
| 1555 |
+
|
| 1556 |
+
db_session = session_factory()
|
| 1557 |
+
try:
|
| 1558 |
+
if user_id:
|
| 1559 |
+
enabled_agents_dict = load_user_enabled_templates_for_planner_from_db(user_id, db_session)
|
| 1560 |
+
db_enabled_agents = list(enabled_agents_dict.keys())
|
| 1561 |
+
else:
|
| 1562 |
+
db_enabled_agents = ["No user_id - using defaults"]
|
| 1563 |
+
finally:
|
| 1564 |
+
db_session.close()
|
| 1565 |
+
|
| 1566 |
+
return {
|
| 1567 |
+
"session_id": session_id,
|
| 1568 |
+
"user_id": user_id,
|
| 1569 |
+
"deep_analyzer_agents": available_agents,
|
| 1570 |
+
"db_enabled_agents": db_enabled_agents,
|
| 1571 |
+
"agents_match": set(available_agents) == set(db_enabled_agents) if user_id else "N/A"
|
| 1572 |
+
}
|
| 1573 |
+
|
| 1574 |
+
except Exception as e:
|
| 1575 |
+
logger.log_message(f"Error in debug endpoint: {str(e)}", level=logging.ERROR)
|
| 1576 |
+
return {
|
| 1577 |
+
"error": str(e),
|
| 1578 |
+
"session_id": session_id,
|
| 1579 |
+
"user_id": user_id
|
| 1580 |
+
}
|
| 1581 |
+
|
| 1582 |
+
@app.post("/debug/clear_deep_analyzer")
|
| 1583 |
+
async def clear_deep_analyzer_cache(session_id: str = Depends(get_session_id_dependency)):
|
| 1584 |
+
"""Debug endpoint to clear the deep analyzer cache and force reload"""
|
| 1585 |
+
session_state = app.state.get_session_state(session_id)
|
| 1586 |
+
|
| 1587 |
+
# Clear the cached deep analyzer
|
| 1588 |
+
if 'deep_analyzer' in session_state:
|
| 1589 |
+
del session_state['deep_analyzer']
|
| 1590 |
+
if 'deep_analyzer_user_id' in session_state:
|
| 1591 |
+
del session_state['deep_analyzer_user_id']
|
| 1592 |
+
|
| 1593 |
+
logger.log_message(f"Cleared deep analyzer cache for session {session_id}", level=logging.INFO)
|
| 1594 |
+
|
| 1595 |
+
return {
|
| 1596 |
+
"message": "Deep analyzer cache cleared",
|
| 1597 |
+
"session_id": session_id,
|
| 1598 |
+
"user_id": session_state.get("user_id")
|
| 1599 |
+
}
|
| 1600 |
+
|
| 1601 |
# In the section where routers are included, add the session_router
|
| 1602 |
app.include_router(chat_router)
|
| 1603 |
app.include_router(analytics_router)
|
chat_database.db
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2ac13830dde4e11eb1ab5b528da9fd6e539205adc6979d53ce7dba040dbd38c5
|
| 3 |
+
size 3137536
|
entrypoint.sh
CHANGED
|
@@ -1,58 +1,89 @@
|
|
| 1 |
#!/bin/bash
|
| 2 |
|
| 3 |
# Entrypoint script for Auto-Analyst backend
|
| 4 |
-
# This script
|
| 5 |
-
#
|
| 6 |
|
| 7 |
set -e # Exit on any error
|
| 8 |
|
| 9 |
echo "🚀 Starting Auto-Analyst Backend..."
|
| 10 |
|
| 11 |
-
# Function to run
|
| 12 |
init_production_database() {
|
| 13 |
-
echo "🔧 Running
|
|
|
|
|
|
|
| 14 |
python scripts/init_production_db.py
|
|
|
|
|
|
|
| 15 |
if [ $? -eq 0 ]; then
|
| 16 |
echo "✅ Database initialization completed successfully"
|
| 17 |
else
|
| 18 |
-
echo "
|
| 19 |
-
|
| 20 |
fi
|
| 21 |
}
|
| 22 |
|
| 23 |
-
# Function to verify
|
| 24 |
-
|
| 25 |
-
echo "
|
| 26 |
python -c "
|
| 27 |
-
import sys
|
| 28 |
-
sys.path.append('.')
|
| 29 |
try:
|
| 30 |
from app import app
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
from src.db.schemas.models import AgentTemplate
|
| 33 |
|
| 34 |
-
|
|
|
|
|
|
|
| 35 |
session = get_session()
|
| 36 |
-
template_count = session.query(AgentTemplate).count()
|
| 37 |
-
session.close()
|
| 38 |
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
except Exception as e:
|
| 41 |
-
print(f'
|
| 42 |
-
|
| 43 |
"
|
|
|
|
| 44 |
}
|
| 45 |
|
| 46 |
# Main startup sequence
|
| 47 |
echo "🔧 Initializing production environment..."
|
| 48 |
|
| 49 |
-
#
|
|
|
|
|
|
|
|
|
|
| 50 |
init_production_database
|
| 51 |
|
| 52 |
-
#
|
| 53 |
-
|
| 54 |
|
| 55 |
echo "🎯 Starting FastAPI application..."
|
|
|
|
| 56 |
|
| 57 |
# Start the FastAPI application
|
| 58 |
exec uvicorn app:app --host 0.0.0.0 --port 7860
|
|
|
|
| 1 |
#!/bin/bash
|
| 2 |
|
| 3 |
# Entrypoint script for Auto-Analyst backend
|
| 4 |
+
# This script safely initializes the database and starts the application
|
| 5 |
+
# SAFE for PostgreSQL/RDS - only modifies SQLite databases
|
| 6 |
|
| 7 |
set -e # Exit on any error
|
| 8 |
|
| 9 |
echo "🚀 Starting Auto-Analyst Backend..."
|
| 10 |
|
| 11 |
+
# Function to run safe database initialization
|
| 12 |
init_production_database() {
|
| 13 |
+
echo "🔧 Running SAFE database initialization..."
|
| 14 |
+
|
| 15 |
+
# Run the safe initialization script
|
| 16 |
python scripts/init_production_db.py
|
| 17 |
+
|
| 18 |
+
# Don't fail if database initialization has issues - let app try to start
|
| 19 |
if [ $? -eq 0 ]; then
|
| 20 |
echo "✅ Database initialization completed successfully"
|
| 21 |
else
|
| 22 |
+
echo "⚠️ Database initialization had issues, but continuing..."
|
| 23 |
+
echo "📋 App will start but some features may not work properly"
|
| 24 |
fi
|
| 25 |
}
|
| 26 |
|
| 27 |
+
# Function to verify basic app imports work
|
| 28 |
+
verify_app_imports() {
|
| 29 |
+
echo "🔍 Verifying application imports..."
|
| 30 |
python -c "
|
|
|
|
|
|
|
| 31 |
try:
|
| 32 |
from app import app
|
| 33 |
+
print('✅ Main application imports successful')
|
| 34 |
+
except Exception as e:
|
| 35 |
+
print(f'❌ Application import failed: {e}')
|
| 36 |
+
exit(1)
|
| 37 |
+
" || {
|
| 38 |
+
echo "❌ Critical application import failure - cannot start"
|
| 39 |
+
exit 1
|
| 40 |
+
}
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
# Function to verify database connectivity (non-failing)
|
| 44 |
+
verify_database_connectivity() {
|
| 45 |
+
echo "🔗 Testing database connectivity..."
|
| 46 |
+
python -c "
|
| 47 |
+
try:
|
| 48 |
+
from src.db.init_db import get_session, is_postgres_db
|
| 49 |
from src.db.schemas.models import AgentTemplate
|
| 50 |
|
| 51 |
+
db_type = 'PostgreSQL/RDS' if is_postgres_db() else 'SQLite'
|
| 52 |
+
print(f'🗄️ Database type: {db_type}')
|
| 53 |
+
|
| 54 |
session = get_session()
|
|
|
|
|
|
|
| 55 |
|
| 56 |
+
# Try to query templates if table exists
|
| 57 |
+
try:
|
| 58 |
+
template_count = session.query(AgentTemplate).count()
|
| 59 |
+
print(f'✅ Database connected. Found {template_count} templates.')
|
| 60 |
+
except Exception as table_error:
|
| 61 |
+
print(f'⚠️ Database connected but template table issue: {table_error}')
|
| 62 |
+
print('📋 Template functionality may not work')
|
| 63 |
+
finally:
|
| 64 |
+
session.close()
|
| 65 |
+
|
| 66 |
except Exception as e:
|
| 67 |
+
print(f'⚠️ Database connectivity issue: {e}')
|
| 68 |
+
print('📋 App will start but database features may not work')
|
| 69 |
"
|
| 70 |
+
# Don't exit on database connectivity issues - let app try to start
|
| 71 |
}
|
| 72 |
|
| 73 |
# Main startup sequence
|
| 74 |
echo "🔧 Initializing production environment..."
|
| 75 |
|
| 76 |
+
# Verify critical imports first
|
| 77 |
+
verify_app_imports
|
| 78 |
+
|
| 79 |
+
# Initialize database safely (won't modify RDS)
|
| 80 |
init_production_database
|
| 81 |
|
| 82 |
+
# Test database connectivity (non-failing)
|
| 83 |
+
verify_database_connectivity
|
| 84 |
|
| 85 |
echo "🎯 Starting FastAPI application..."
|
| 86 |
+
echo "🌐 Application will be available on port 7860"
|
| 87 |
|
| 88 |
# Start the FastAPI application
|
| 89 |
exec uvicorn app:app --host 0.0.0.0 --port 7860
|
heatmap_correlation.png
ADDED
|
Git LFS Details
|
load_default_agents.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Script to load default agents into the AgentTemplate table.
|
| 4 |
+
Run this script to populate the database with the default agents.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import sys
|
| 8 |
+
import os
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
# Add the src directory to the path
|
| 12 |
+
src_path = Path(__file__).parent / "src"
|
| 13 |
+
sys.path.append(str(src_path))
|
| 14 |
+
|
| 15 |
+
def main():
|
| 16 |
+
try:
|
| 17 |
+
from src.db.init_default_agents import initialize_default_agents
|
| 18 |
+
|
| 19 |
+
# Initialize default agents with force update enabled
|
| 20 |
+
success = initialize_default_agents(force_update=True)
|
| 21 |
+
|
| 22 |
+
if success:
|
| 23 |
+
print("✅ Successfully loaded default agents into the database!")
|
| 24 |
+
print("The following agents are now available:")
|
| 25 |
+
print(" • Data Preprocessing Agent (preprocessing_agent)")
|
| 26 |
+
print(" • Statistical Analytics Agent (statistical_analytics_agent)")
|
| 27 |
+
print(" • Machine Learning Agent (sk_learn_agent)")
|
| 28 |
+
print(" • Data Visualization Agent (data_viz_agent)")
|
| 29 |
+
else:
|
| 30 |
+
print("❌ Failed to load default agents")
|
| 31 |
+
sys.exit(1)
|
| 32 |
+
|
| 33 |
+
except ImportError as e:
|
| 34 |
+
print(f"❌ Import error: {e}")
|
| 35 |
+
print("Make sure you're running this script from the auto-analyst-backend directory")
|
| 36 |
+
sys.exit(1)
|
| 37 |
+
except Exception as e:
|
| 38 |
+
print(f"❌ Error: {e}")
|
| 39 |
+
sys.exit(1)
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
main()
|
scripts/init_production_db.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
"""
|
| 3 |
Production database initialization script.
|
| 4 |
-
This ensures
|
|
|
|
| 5 |
"""
|
| 6 |
|
| 7 |
import sys
|
|
@@ -12,7 +13,7 @@ from datetime import datetime, UTC
|
|
| 12 |
# Add the project root to the Python path
|
| 13 |
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 14 |
|
| 15 |
-
from src.db.init_db import init_db, session_factory, engine
|
| 16 |
from src.db.schemas.models import Base, AgentTemplate, UserTemplatePreference
|
| 17 |
from scripts.populate_agent_templates import populate_templates
|
| 18 |
from sqlalchemy import inspect, text
|
|
@@ -20,6 +21,17 @@ from src.utils.logger import Logger
|
|
| 20 |
|
| 21 |
logger = Logger("init_production_db", see_time=True, console_log=True)
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
def check_table_exists(table_name: str) -> bool:
|
| 24 |
"""Check if a table exists in the database."""
|
| 25 |
try:
|
|
@@ -31,8 +43,9 @@ def check_table_exists(table_name: str) -> bool:
|
|
| 31 |
return False
|
| 32 |
|
| 33 |
def verify_database_schema():
|
| 34 |
-
"""Verify that all required tables exist
|
| 35 |
-
|
|
|
|
| 36 |
|
| 37 |
required_tables = [
|
| 38 |
'users', 'chats', 'messages', 'model_usage', 'code_executions',
|
|
@@ -41,56 +54,85 @@ def verify_database_schema():
|
|
| 41 |
]
|
| 42 |
|
| 43 |
missing_tables = []
|
|
|
|
|
|
|
| 44 |
for table in required_tables:
|
| 45 |
if not check_table_exists(table):
|
| 46 |
missing_tables.append(table)
|
| 47 |
logger.log_message(f"❌ Missing table: {table}", logging.WARNING)
|
| 48 |
else:
|
|
|
|
| 49 |
logger.log_message(f"✅ Table exists: {table}", logging.INFO)
|
| 50 |
|
| 51 |
if missing_tables:
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
else:
|
| 61 |
-
logger.log_message("✅ All required tables exist", logging.INFO)
|
| 62 |
|
| 63 |
def verify_template_data():
|
| 64 |
-
"""Verify that agent templates are populated."""
|
| 65 |
logger.log_message("📋 Verifying template data...", logging.INFO)
|
| 66 |
|
| 67 |
session = session_factory()
|
| 68 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
template_count = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).count()
|
| 70 |
logger.log_message(f"📊 Found {template_count} active templates", logging.INFO)
|
| 71 |
|
| 72 |
if template_count == 0:
|
| 73 |
logger.log_message("🔧 No templates found, populating...", logging.INFO)
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
else:
|
| 80 |
logger.log_message("✅ Templates already populated", logging.INFO)
|
| 81 |
|
| 82 |
except Exception as e:
|
| 83 |
logger.log_message(f"❌ Error verifying templates: {e}", logging.ERROR)
|
| 84 |
-
|
| 85 |
finally:
|
| 86 |
session.close()
|
| 87 |
|
| 88 |
def test_template_api_functionality():
|
| 89 |
-
"""Test that template-related database operations work."""
|
| 90 |
logger.log_message("🧪 Testing template API functionality...", logging.INFO)
|
| 91 |
|
| 92 |
session = session_factory()
|
| 93 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
# Test basic template query
|
| 95 |
templates = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).limit(5).all()
|
| 96 |
logger.log_message(f"✅ Successfully queried {len(templates)} templates", logging.INFO)
|
|
@@ -98,39 +140,52 @@ def test_template_api_functionality():
|
|
| 98 |
if templates:
|
| 99 |
sample_template = templates[0]
|
| 100 |
logger.log_message(f"📄 Sample template: {sample_template.template_name} - {sample_template.display_name}", logging.INFO)
|
|
|
|
|
|
|
| 101 |
|
| 102 |
except Exception as e:
|
| 103 |
logger.log_message(f"❌ Template API test failed: {e}", logging.ERROR)
|
| 104 |
-
|
| 105 |
finally:
|
| 106 |
session.close()
|
| 107 |
|
| 108 |
-
def
|
| 109 |
-
"""Run
|
| 110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
|
| 112 |
try:
|
| 113 |
-
# Step 1: Initialize database
|
| 114 |
-
logger.log_message("Step 1:
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
-
# Step 2: Verify schema
|
| 118 |
logger.log_message("Step 2: Schema verification", logging.INFO)
|
| 119 |
verify_database_schema()
|
| 120 |
|
| 121 |
-
# Step 3: Verify template data
|
| 122 |
logger.log_message("Step 3: Template data verification", logging.INFO)
|
| 123 |
verify_template_data()
|
| 124 |
|
| 125 |
-
# Step 4: Test functionality
|
| 126 |
logger.log_message("Step 4: Functionality testing", logging.INFO)
|
| 127 |
test_template_api_functionality()
|
| 128 |
|
| 129 |
-
logger.log_message("🎉
|
| 130 |
|
| 131 |
except Exception as e:
|
| 132 |
logger.log_message(f"💥 Database initialization failed: {e}", logging.ERROR)
|
| 133 |
-
|
|
|
|
| 134 |
|
| 135 |
if __name__ == "__main__":
|
| 136 |
-
|
|
|
|
| 1 |
#!/usr/bin/env python3
|
| 2 |
"""
|
| 3 |
Production database initialization script.
|
| 4 |
+
This ensures templates are populated properly and verifies database health.
|
| 5 |
+
SAFE for PostgreSQL/RDS - only creates tables on SQLite databases.
|
| 6 |
"""
|
| 7 |
|
| 8 |
import sys
|
|
|
|
| 13 |
# Add the project root to the Python path
|
| 14 |
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 15 |
|
| 16 |
+
from src.db.init_db import init_db, session_factory, engine, is_postgres_db
|
| 17 |
from src.db.schemas.models import Base, AgentTemplate, UserTemplatePreference
|
| 18 |
from scripts.populate_agent_templates import populate_templates
|
| 19 |
from sqlalchemy import inspect, text
|
|
|
|
| 21 |
|
| 22 |
logger = Logger("init_production_db", see_time=True, console_log=True)
|
| 23 |
|
| 24 |
+
def get_database_type():
|
| 25 |
+
"""Get the database type (sqlite or postgresql)."""
|
| 26 |
+
try:
|
| 27 |
+
if is_postgres_db():
|
| 28 |
+
return "postgresql"
|
| 29 |
+
else:
|
| 30 |
+
return "sqlite"
|
| 31 |
+
except Exception as e:
|
| 32 |
+
logger.log_message(f"Error determining database type: {e}", logging.ERROR)
|
| 33 |
+
return "unknown"
|
| 34 |
+
|
| 35 |
def check_table_exists(table_name: str) -> bool:
|
| 36 |
"""Check if a table exists in the database."""
|
| 37 |
try:
|
|
|
|
| 43 |
return False
|
| 44 |
|
| 45 |
def verify_database_schema():
|
| 46 |
+
"""Verify that all required tables exist. Only create tables on SQLite."""
|
| 47 |
+
db_type = get_database_type()
|
| 48 |
+
logger.log_message(f"🔍 Verifying database schema for {db_type.upper()} database...", logging.INFO)
|
| 49 |
|
| 50 |
required_tables = [
|
| 51 |
'users', 'chats', 'messages', 'model_usage', 'code_executions',
|
|
|
|
| 54 |
]
|
| 55 |
|
| 56 |
missing_tables = []
|
| 57 |
+
existing_tables = []
|
| 58 |
+
|
| 59 |
for table in required_tables:
|
| 60 |
if not check_table_exists(table):
|
| 61 |
missing_tables.append(table)
|
| 62 |
logger.log_message(f"❌ Missing table: {table}", logging.WARNING)
|
| 63 |
else:
|
| 64 |
+
existing_tables.append(table)
|
| 65 |
logger.log_message(f"✅ Table exists: {table}", logging.INFO)
|
| 66 |
|
| 67 |
if missing_tables:
|
| 68 |
+
if db_type == "sqlite":
|
| 69 |
+
logger.log_message(f"🔧 Creating missing tables on SQLite: {missing_tables}", logging.INFO)
|
| 70 |
+
try:
|
| 71 |
+
# Safe to create tables on SQLite
|
| 72 |
+
Base.metadata.create_all(engine)
|
| 73 |
+
logger.log_message("✅ All tables created successfully on SQLite", logging.INFO)
|
| 74 |
+
except Exception as e:
|
| 75 |
+
logger.log_message(f"❌ Failed to create tables: {e}", logging.ERROR)
|
| 76 |
+
raise
|
| 77 |
+
else:
|
| 78 |
+
# PostgreSQL/RDS - DO NOT create tables automatically
|
| 79 |
+
logger.log_message(f"⚠️ WARNING: Missing tables detected in {db_type.upper()} database: {missing_tables}", logging.WARNING)
|
| 80 |
+
logger.log_message("🛡️ SAFETY: Not creating tables automatically on PostgreSQL/RDS", logging.INFO)
|
| 81 |
+
logger.log_message("📋 Please ensure these tables exist in your RDS database:", logging.INFO)
|
| 82 |
+
for table in missing_tables:
|
| 83 |
+
logger.log_message(f" - {table}", logging.INFO)
|
| 84 |
+
|
| 85 |
+
# Continue without failing - the app might still work with existing tables
|
| 86 |
+
if 'agent_templates' in missing_tables or 'user_template_preferences' in missing_tables:
|
| 87 |
+
logger.log_message("⚠️ Template functionality may not work without agent_templates and user_template_preferences tables", logging.WARNING)
|
| 88 |
else:
|
| 89 |
+
logger.log_message(f"✅ All required tables exist in {db_type.upper()} database", logging.INFO)
|
| 90 |
|
| 91 |
def verify_template_data():
|
| 92 |
+
"""Verify that agent templates are populated. Safe for all database types."""
|
| 93 |
logger.log_message("📋 Verifying template data...", logging.INFO)
|
| 94 |
|
| 95 |
session = session_factory()
|
| 96 |
try:
|
| 97 |
+
# Check if agent_templates table exists before querying
|
| 98 |
+
if not check_table_exists('agent_templates'):
|
| 99 |
+
logger.log_message("⚠️ agent_templates table does not exist, skipping template verification", logging.WARNING)
|
| 100 |
+
return
|
| 101 |
+
|
| 102 |
template_count = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).count()
|
| 103 |
logger.log_message(f"📊 Found {template_count} active templates", logging.INFO)
|
| 104 |
|
| 105 |
if template_count == 0:
|
| 106 |
logger.log_message("🔧 No templates found, populating...", logging.INFO)
|
| 107 |
+
try:
|
| 108 |
+
populate_templates()
|
| 109 |
+
|
| 110 |
+
# Verify population worked
|
| 111 |
+
new_count = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).count()
|
| 112 |
+
logger.log_message(f"✅ Templates populated. Total active templates: {new_count}", logging.INFO)
|
| 113 |
+
except Exception as e:
|
| 114 |
+
logger.log_message(f"❌ Template population failed: {e}", logging.ERROR)
|
| 115 |
+
logger.log_message("⚠️ App will continue but template functionality may not work", logging.WARNING)
|
| 116 |
else:
|
| 117 |
logger.log_message("✅ Templates already populated", logging.INFO)
|
| 118 |
|
| 119 |
except Exception as e:
|
| 120 |
logger.log_message(f"❌ Error verifying templates: {e}", logging.ERROR)
|
| 121 |
+
logger.log_message("⚠️ Template verification failed, but app will continue", logging.WARNING)
|
| 122 |
finally:
|
| 123 |
session.close()
|
| 124 |
|
| 125 |
def test_template_api_functionality():
|
| 126 |
+
"""Test that template-related database operations work. Safe for all database types."""
|
| 127 |
logger.log_message("🧪 Testing template API functionality...", logging.INFO)
|
| 128 |
|
| 129 |
session = session_factory()
|
| 130 |
try:
|
| 131 |
+
# Check if agent_templates table exists before testing
|
| 132 |
+
if not check_table_exists('agent_templates'):
|
| 133 |
+
logger.log_message("⚠️ agent_templates table does not exist, skipping API test", logging.WARNING)
|
| 134 |
+
return
|
| 135 |
+
|
| 136 |
# Test basic template query
|
| 137 |
templates = session.query(AgentTemplate).filter(AgentTemplate.is_active == True).limit(5).all()
|
| 138 |
logger.log_message(f"✅ Successfully queried {len(templates)} templates", logging.INFO)
|
|
|
|
| 140 |
if templates:
|
| 141 |
sample_template = templates[0]
|
| 142 |
logger.log_message(f"📄 Sample template: {sample_template.template_name} - {sample_template.display_name}", logging.INFO)
|
| 143 |
+
else:
|
| 144 |
+
logger.log_message("📭 No templates found in database", logging.INFO)
|
| 145 |
|
| 146 |
except Exception as e:
|
| 147 |
logger.log_message(f"❌ Template API test failed: {e}", logging.ERROR)
|
| 148 |
+
logger.log_message("⚠️ Template API may not work properly", logging.WARNING)
|
| 149 |
finally:
|
| 150 |
session.close()
|
| 151 |
|
| 152 |
+
def run_safe_initialization():
|
| 153 |
+
"""Run safe database initialization that respects production databases."""
|
| 154 |
+
db_type = get_database_type()
|
| 155 |
+
logger.log_message(f"🚀 Starting SAFE database initialization for {db_type.upper()}...", logging.INFO)
|
| 156 |
+
|
| 157 |
+
if db_type == "postgresql":
|
| 158 |
+
logger.log_message("🛡️ PostgreSQL/RDS detected - running in SAFE mode", logging.INFO)
|
| 159 |
+
logger.log_message("📋 Will only verify schema and populate templates", logging.INFO)
|
| 160 |
+
elif db_type == "sqlite":
|
| 161 |
+
logger.log_message("💽 SQLite detected - full initialization mode", logging.INFO)
|
| 162 |
|
| 163 |
try:
|
| 164 |
+
# Step 1: Initialize database (safe for all types)
|
| 165 |
+
logger.log_message("Step 1: Basic database initialization", logging.INFO)
|
| 166 |
+
if db_type == "sqlite":
|
| 167 |
+
init_db() # Only run full init on SQLite
|
| 168 |
+
else:
|
| 169 |
+
logger.log_message("Skipping init_db() for PostgreSQL (safety)", logging.INFO)
|
| 170 |
|
| 171 |
+
# Step 2: Verify schema (safe - only creates tables on SQLite)
|
| 172 |
logger.log_message("Step 2: Schema verification", logging.INFO)
|
| 173 |
verify_database_schema()
|
| 174 |
|
| 175 |
+
# Step 3: Verify template data (safe for all types)
|
| 176 |
logger.log_message("Step 3: Template data verification", logging.INFO)
|
| 177 |
verify_template_data()
|
| 178 |
|
| 179 |
+
# Step 4: Test functionality (safe for all types)
|
| 180 |
logger.log_message("Step 4: Functionality testing", logging.INFO)
|
| 181 |
test_template_api_functionality()
|
| 182 |
|
| 183 |
+
logger.log_message(f"🎉 Safe database initialization completed for {db_type.upper()}!", logging.INFO)
|
| 184 |
|
| 185 |
except Exception as e:
|
| 186 |
logger.log_message(f"💥 Database initialization failed: {e}", logging.ERROR)
|
| 187 |
+
logger.log_message("⚠️ App may still start but some features might not work", logging.WARNING)
|
| 188 |
+
# Don't raise - let the app try to start anyway
|
| 189 |
|
| 190 |
if __name__ == "__main__":
|
| 191 |
+
run_safe_initialization()
|
scripts/populate_agent_templates.py
CHANGED
|
@@ -22,7 +22,7 @@ AGENT_TEMPLATES = {
|
|
| 22 |
"template_name": "matplotlib_agent",
|
| 23 |
"display_name": "Matplotlib Visualization Agent",
|
| 24 |
"description": "Creates static publication-quality plots using matplotlib and seaborn",
|
| 25 |
-
"icon_url": "
|
| 26 |
"prompt_template": """
|
| 27 |
You are a matplotlib/seaborn visualization expert. Your task is to create high-quality static visualizations using matplotlib and seaborn libraries.
|
| 28 |
|
|
@@ -42,7 +42,7 @@ Focus on creating publication-ready static visualizations that are informative a
|
|
| 42 |
"template_name": "seaborn_agent",
|
| 43 |
"display_name": "Seaborn Statistical Plots Agent",
|
| 44 |
"description": "Creates statistical visualizations and data exploration plots using seaborn",
|
| 45 |
-
"icon_url": "
|
| 46 |
"prompt_template": """
|
| 47 |
You are a seaborn statistical visualization expert. Your task is to create statistical plots and exploratory data visualizations.
|
| 48 |
|
|
@@ -65,7 +65,7 @@ Focus on revealing statistical patterns and relationships in data through visual
|
|
| 65 |
"template_name": "polars_agent",
|
| 66 |
"display_name": "Polars Data Processing Agent",
|
| 67 |
"description": "High-performance data manipulation and analysis using Polars",
|
| 68 |
-
"icon_url": "
|
| 69 |
"prompt_template": """
|
| 70 |
You are a Polars data processing expert. Perform high-performance data manipulation and analysis using Polars.
|
| 71 |
|
|
@@ -88,7 +88,7 @@ Focus on leveraging Polars' speed and efficiency for data processing tasks.
|
|
| 88 |
"template_name": "data_cleaning_agent",
|
| 89 |
"display_name": "Data Cleaning Specialist Agent",
|
| 90 |
"description": "Specialized in comprehensive data cleaning and quality assessment",
|
| 91 |
-
"icon_url": "
|
| 92 |
"prompt_template": """
|
| 93 |
You are a data cleaning specialist. Perform comprehensive data quality assessment and cleaning.
|
| 94 |
|
|
@@ -111,7 +111,7 @@ Focus on delivering high-quality, analysis-ready datasets with comprehensive doc
|
|
| 111 |
"template_name": "feature_engineering_agent",
|
| 112 |
"display_name": "Feature Engineering Agent",
|
| 113 |
"description": "Creates and transforms features for machine learning models",
|
| 114 |
-
"icon_url": "
|
| 115 |
"prompt_template": """
|
| 116 |
You are a feature engineering expert. Create, transform, and select features for machine learning.
|
| 117 |
|
|
|
|
| 22 |
"template_name": "matplotlib_agent",
|
| 23 |
"display_name": "Matplotlib Visualization Agent",
|
| 24 |
"description": "Creates static publication-quality plots using matplotlib and seaborn",
|
| 25 |
+
"icon_url": "/icons/templates/matplotlib.svg",
|
| 26 |
"prompt_template": """
|
| 27 |
You are a matplotlib/seaborn visualization expert. Your task is to create high-quality static visualizations using matplotlib and seaborn libraries.
|
| 28 |
|
|
|
|
| 42 |
"template_name": "seaborn_agent",
|
| 43 |
"display_name": "Seaborn Statistical Plots Agent",
|
| 44 |
"description": "Creates statistical visualizations and data exploration plots using seaborn",
|
| 45 |
+
"icon_url": "/icons/templates/seaborn.svg",
|
| 46 |
"prompt_template": """
|
| 47 |
You are a seaborn statistical visualization expert. Your task is to create statistical plots and exploratory data visualizations.
|
| 48 |
|
|
|
|
| 65 |
"template_name": "polars_agent",
|
| 66 |
"display_name": "Polars Data Processing Agent",
|
| 67 |
"description": "High-performance data manipulation and analysis using Polars",
|
| 68 |
+
"icon_url": "/icons/templates/polars.svg",
|
| 69 |
"prompt_template": """
|
| 70 |
You are a Polars data processing expert. Perform high-performance data manipulation and analysis using Polars.
|
| 71 |
|
|
|
|
| 88 |
"template_name": "data_cleaning_agent",
|
| 89 |
"display_name": "Data Cleaning Specialist Agent",
|
| 90 |
"description": "Specialized in comprehensive data cleaning and quality assessment",
|
| 91 |
+
"icon_url": "/icons/templates/data-cleaning.png",
|
| 92 |
"prompt_template": """
|
| 93 |
You are a data cleaning specialist. Perform comprehensive data quality assessment and cleaning.
|
| 94 |
|
|
|
|
| 111 |
"template_name": "feature_engineering_agent",
|
| 112 |
"display_name": "Feature Engineering Agent",
|
| 113 |
"description": "Creates and transforms features for machine learning models",
|
| 114 |
+
"icon_url": "/icons/templates/feature-engineering.png",
|
| 115 |
"prompt_template": """
|
| 116 |
You are a feature engineering expert. Create, transform, and select features for machine learning.
|
| 117 |
|
src/agents/agents.py
CHANGED
|
@@ -22,16 +22,23 @@ def create_custom_agent_signature(agent_name, description, prompt_template):
|
|
| 22 |
A dspy.Signature class with the custom prompt and standard input/output fields
|
| 23 |
"""
|
| 24 |
|
| 25 |
-
#
|
|
|
|
|
|
|
|
|
|
| 26 |
class_attributes = {
|
| 27 |
'__doc__': prompt_template, # The custom prompt becomes the docstring
|
| 28 |
'goal': dspy.InputField(desc="User-defined goal which includes information about data and task they want to perform"),
|
| 29 |
'dataset': dspy.InputField(desc="Provides information about the data in the data frame. Only use column names and dataframe_name as in this context"),
|
| 30 |
-
'
|
| 31 |
'code': dspy.OutputField(desc="Generated Python code for the analysis"),
|
| 32 |
'summary': dspy.OutputField(desc="A concise bullet-point summary of what was done and key results")
|
| 33 |
}
|
| 34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
# Create the dynamic signature class
|
| 36 |
CustomAgentSignature = type(agent_name, (dspy.Signature,), class_attributes)
|
| 37 |
return CustomAgentSignature
|
|
@@ -39,7 +46,7 @@ def create_custom_agent_signature(agent_name, description, prompt_template):
|
|
| 39 |
def load_user_enabled_templates_from_db(user_id, db_session):
|
| 40 |
"""
|
| 41 |
Load template agents that are enabled for a specific user from the database.
|
| 42 |
-
|
| 43 |
|
| 44 |
Args:
|
| 45 |
user_id: ID of the user
|
|
@@ -56,6 +63,14 @@ def load_user_enabled_templates_from_db(user_id, db_session):
|
|
| 56 |
if not user_id:
|
| 57 |
return agent_signatures
|
| 58 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
# Get all active templates
|
| 60 |
all_templates = db_session.query(AgentTemplate).filter(
|
| 61 |
AgentTemplate.is_active == True
|
|
@@ -68,10 +83,13 @@ def load_user_enabled_templates_from_db(user_id, db_session):
|
|
| 68 |
UserTemplatePreference.template_id == template.template_id
|
| 69 |
).first()
|
| 70 |
|
| 71 |
-
#
|
| 72 |
-
|
| 73 |
-
|
| 74 |
|
|
|
|
|
|
|
|
|
|
| 75 |
if is_enabled:
|
| 76 |
# Create dynamic signature for each enabled template
|
| 77 |
signature = create_custom_agent_signature(
|
|
@@ -90,6 +108,7 @@ def load_user_enabled_templates_from_db(user_id, db_session):
|
|
| 90 |
def load_user_enabled_templates_for_planner_from_db(user_id, db_session):
|
| 91 |
"""
|
| 92 |
Load template agents that are enabled for planner use (max 10, prioritized by usage).
|
|
|
|
| 93 |
|
| 94 |
Args:
|
| 95 |
user_id: ID of the user
|
|
@@ -100,36 +119,62 @@ def load_user_enabled_templates_for_planner_from_db(user_id, db_session):
|
|
| 100 |
"""
|
| 101 |
try:
|
| 102 |
from src.db.schemas.models import AgentTemplate, UserTemplatePreference
|
|
|
|
| 103 |
|
| 104 |
agent_signatures = {}
|
| 105 |
|
| 106 |
if not user_id:
|
| 107 |
return agent_signatures
|
| 108 |
|
| 109 |
-
# Get
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
).limit(10).all()
|
| 117 |
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
).first()
|
| 124 |
|
| 125 |
-
if template
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
logger.log_message(f"Loaded {len(agent_signatures)} templates for planner", level=logging.DEBUG)
|
| 135 |
return agent_signatures
|
|
@@ -251,7 +296,6 @@ def load_all_available_templates_from_db(db_session):
|
|
| 251 |
)
|
| 252 |
agent_signatures[template.template_name] = signature
|
| 253 |
|
| 254 |
-
logger.log_message(f"Loaded {len(agent_signatures)} templates", level=logging.INFO)
|
| 255 |
return agent_signatures
|
| 256 |
|
| 257 |
except Exception as e:
|
|
@@ -262,43 +306,30 @@ def load_all_available_templates_from_db(db_session):
|
|
| 262 |
|
| 263 |
# === END CUSTOM AGENT FUNCTIONALITY ===
|
| 264 |
|
| 265 |
-
AGENTS_WITH_DESCRIPTION = {
|
| 266 |
-
"preprocessing_agent": "Cleans and prepares a DataFrame using Pandas and NumPy—handles missing values, detects column types, and converts date strings to datetime.",
|
| 267 |
-
"statistical_analytics_agent": "Performs statistical analysis (e.g., regression, seasonal decomposition) using statsmodels, with proper handling of categorical data and missing values.",
|
| 268 |
-
"sk_learn_agent": "Trains and evaluates machine learning models using scikit-learn, including classification, regression, and clustering with feature importance insights.",
|
| 269 |
-
"data_viz_agent": "Generates interactive visualizations with Plotly, selecting the best chart type to reveal trends, comparisons, and insights based on the analysis goal."
|
| 270 |
-
}
|
| 271 |
-
|
| 272 |
-
PLANNER_AGENTS_WITH_DESCRIPTION = {
|
| 273 |
-
"planner_preprocessing_agent": (
|
| 274 |
-
"Cleans and prepares a DataFrame using Pandas and NumPy"
|
| 275 |
-
"handles missing values, detects column types, and converts date strings to datetime. "
|
| 276 |
-
"Outputs a cleaned DataFrame for the planner_statistical_analytics_agent."
|
| 277 |
-
),
|
| 278 |
-
"planner_statistical_analytics_agent": (
|
| 279 |
-
"Takes the cleaned DataFrame from preprocessing, performs statistical analysis "
|
| 280 |
-
"(e.g., regression, seasonal decomposition) using statsmodels with proper handling "
|
| 281 |
-
"of categorical data and remaining missing values. "
|
| 282 |
-
"Produces summary statistics and model diagnostics for the planner_sk_learn_agent."
|
| 283 |
-
),
|
| 284 |
-
"planner_sk_learn_agent": (
|
| 285 |
-
"Receives summary statistics and the cleaned data, trains and evaluates machine "
|
| 286 |
-
"learning models using scikit-learn (classification, regression, clustering), "
|
| 287 |
-
"and generates performance metrics and feature importance. "
|
| 288 |
-
"Passes the trained models and evaluation results to the planner_data_viz_agent."
|
| 289 |
-
),
|
| 290 |
-
"planner_data_viz_agent": (
|
| 291 |
-
"Consumes trained models and evaluation results to create interactive visualizations "
|
| 292 |
-
"with Plotly—selects the best chart type, applies styling, and annotates insights. "
|
| 293 |
-
"Delivers ready-to-share figures that communicate model performance and key findings."
|
| 294 |
-
),
|
| 295 |
-
}
|
| 296 |
-
|
| 297 |
def get_agent_description(agent_name, is_planner=False):
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
|
| 303 |
|
| 304 |
# Agent to make a Chat history name from a query
|
|
@@ -421,6 +452,11 @@ class custom_agent_instruction_generator(dspy.Signature):
|
|
| 421 |
class advanced_query_planner(dspy.Signature):
|
| 422 |
"""
|
| 423 |
You are a advanced data analytics planner agent. Your task is to generate the most efficient plan—using the fewest necessary agents and variables—to achieve a user-defined goal. The plan must preserve data integrity, avoid unnecessary steps, and ensure clear data flow between agents.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 424 |
**Inputs**:
|
| 425 |
1. Datasets (raw or preprocessed)
|
| 426 |
2. Agent descriptions (roles, variables they create/use, constraints)
|
|
@@ -437,9 +473,9 @@ You are a advanced data analytics planner agent. Your task is to generate the mo
|
|
| 437 |
Example: 1 agent use
|
| 438 |
goal: "Generate a bar plot showing sales by category after cleaning the raw data and calculating the average of the 'sales' column"
|
| 439 |
Output:
|
| 440 |
-
plan:
|
| 441 |
{
|
| 442 |
-
"
|
| 443 |
"create": [
|
| 444 |
"cleaned_data: DataFrame - cleaned version of df (pd.Dataframe) after removing null values"
|
| 445 |
],
|
|
@@ -451,9 +487,9 @@ Output:
|
|
| 451 |
}
|
| 452 |
Example 3 Agent
|
| 453 |
goal:"Clean the dataset, run a linear regression to model the relationship between marketing budget and sales, and visualize the regression line with confidence intervals."
|
| 454 |
-
plan:
|
| 455 |
{
|
| 456 |
-
"
|
| 457 |
"create": [
|
| 458 |
"cleaned_data: DataFrame - cleaned version of df with missing values handled and proper data types inferred"
|
| 459 |
],
|
|
@@ -462,7 +498,7 @@ plan: planner_preprocessing_agent -> planner_statistical_analytics_agent -> plan
|
|
| 462 |
],
|
| 463 |
"instruction": "Clean df by handling missing values and converting column types (e.g., dates). Output cleaned_data for modeling."
|
| 464 |
},
|
| 465 |
-
"
|
| 466 |
"create": [
|
| 467 |
"regression_results: dict - model summary including coefficients, p-values, R², and confidence intervals"
|
| 468 |
],
|
|
@@ -471,7 +507,7 @@ plan: planner_preprocessing_agent -> planner_statistical_analytics_agent -> plan
|
|
| 471 |
],
|
| 472 |
"instruction": "Perform linear regression using cleaned_data to model sales as a function of marketing budget. Return regression_results including coefficients and confidence intervals."
|
| 473 |
},
|
| 474 |
-
"
|
| 475 |
"create": [
|
| 476 |
"regression_plot: PlotlyFigure - visual plot showing regression line with confidence intervals"
|
| 477 |
],
|
|
@@ -496,20 +532,25 @@ class basic_query_planner(dspy.Signature):
|
|
| 496 |
"""
|
| 497 |
You are the basic query planner in the system, you pick one agent, to answer the user's goal.
|
| 498 |
Use the Agent_desc that describes the names and actions of agents available.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 499 |
Example: Visualize height and salary?
|
| 500 |
-
plan:
|
| 501 |
plan_instructions:
|
| 502 |
{
|
| 503 |
-
"
|
| 504 |
"create": ["scatter_plot"],
|
| 505 |
"use": ["original_data"],
|
| 506 |
"instruction": "use the original_data to create scatter_plot of height & salary, using plotly"
|
| 507 |
}
|
| 508 |
}
|
| 509 |
Example: Tell me the correlation between X and Y
|
| 510 |
-
plan:
|
| 511 |
plan_instructions:{
|
| 512 |
-
"
|
| 513 |
"create": ["correlation"],
|
| 514 |
"use": ["original_data"],
|
| 515 |
"instruction": "use the original_data to measure correlation of X & Y, using pandas"
|
|
@@ -535,6 +576,11 @@ class intermediate_query_planner(dspy.Signature):
|
|
| 535 |
3. User-defined Goal
|
| 536 |
You take these three inputs to develop a comprehensive plan to achieve the user-defined goal from the data & Agents available.
|
| 537 |
In case you think the user-defined goal is infeasible you can ask the user to redefine or add more description to the goal.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 538 |
Give your output in this format:
|
| 539 |
plan: Agent1->Agent2
|
| 540 |
plan_instructions = {
|
|
@@ -583,53 +629,115 @@ class planner_module(dspy.Module):
|
|
| 583 |
|
| 584 |
self.allocator = dspy.Predict("goal,planner_desc,dataset->exact_word_complexity,reasoning")
|
| 585 |
|
| 586 |
-
async def forward(self, goal,dataset,Agent_desc):
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 590 |
try:
|
|
|
|
| 591 |
plan = await self.planners[complexity.exact_word_complexity.strip()](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 592 |
|
| 593 |
except Exception as e:
|
|
|
|
|
|
|
|
|
|
| 594 |
plan = await self.planners["intermediate"](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 603 |
return output
|
| 604 |
|
| 605 |
|
| 606 |
|
| 607 |
|
| 608 |
|
| 609 |
-
class
|
| 610 |
"""
|
| 611 |
-
You are a preprocessing agent
|
| 612 |
You are given:
|
| 613 |
-
* A
|
| 614 |
-
* A
|
| 615 |
-
*
|
| 616 |
-
|
| 617 |
### Your Responsibilities:
|
| 618 |
-
*
|
| 619 |
-
*
|
| 620 |
-
*
|
| 621 |
-
*
|
|
|
|
|
|
|
| 622 |
### Best Practices for Preprocessing:
|
| 623 |
-
1.
|
| 624 |
```python
|
| 625 |
processed_df = df.copy()
|
| 626 |
```
|
| 627 |
-
2.
|
| 628 |
```python
|
| 629 |
numeric_cols = processed_df.select_dtypes(include='number').columns
|
| 630 |
categorical_cols = processed_df.select_dtypes(include='object').columns
|
| 631 |
```
|
| 632 |
-
3.
|
| 633 |
```python
|
| 634 |
for col in numeric_cols:
|
| 635 |
processed_df[col] = processed_df[col].fillna(processed_df[col].median())
|
|
@@ -637,7 +745,7 @@ You are given:
|
|
| 637 |
for col in categorical_cols:
|
| 638 |
processed_df[col] = processed_df[col].fillna(processed_df[col].mode()[0] if not processed_df[col].mode().empty else 'Unknown')
|
| 639 |
```
|
| 640 |
-
4.
|
| 641 |
```python
|
| 642 |
def safe_to_datetime(x):
|
| 643 |
try:
|
|
@@ -647,138 +755,141 @@ You are given:
|
|
| 647 |
|
| 648 |
cleaned_df['date_column'] = cleaned_df['date_column'].apply(safe_to_datetime)
|
| 649 |
```
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
7. Do not mutate global state : Avoid in-place modifications unless clearly necessary (e.g., using `.copy()`).
|
| 655 |
-
8. Handle data types properly :
|
| 656 |
* Avoid coercing types blindly (e.g., don't compare timestamps to strings or floats).
|
| 657 |
* Use `pd.to_datetime(..., errors='coerce')` for safe datetime parsing.
|
| 658 |
-
9.
|
|
|
|
| 659 |
### Output:
|
| 660 |
-
1.
|
| 661 |
-
2.
|
|
|
|
| 662 |
### Principles to Follow:
|
| 663 |
-
-Never alter the DataFrame index
|
| 664 |
-
-Handle missing data
|
| 665 |
-
-Preserve column structure
|
| 666 |
-
-Ensure data types are appropriate
|
| 667 |
-
-Log assumptions
|
| 668 |
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 669 |
"""
|
| 670 |
dataset = dspy.InputField(desc="The dataset, preloaded as df")
|
| 671 |
goal = dspy.InputField(desc="User-defined goal for the analysis")
|
| 672 |
-
plan_instructions = dspy.InputField(desc="Agent-level instructions about what to create and receive")
|
| 673 |
|
| 674 |
code = dspy.OutputField(desc="Generated Python code for preprocessing")
|
| 675 |
summary = dspy.OutputField(desc="Explanation of what was done and why")
|
| 676 |
|
| 677 |
-
class
|
| 678 |
"""
|
| 679 |
-
|
| 680 |
-
|
|
|
|
| 681 |
You are provided with:
|
| 682 |
* **goal**: A user-defined goal outlining the type of visualization the user wants (e.g., "plot sales over time with trendline").
|
| 683 |
-
|
| 684 |
* **styling_index**: Specific styling instructions (e.g., axis formatting, color schemes) for the visualization.
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
1. **Strict Use of Provided Variables**:
|
| 692 |
-
|
| 693 |
-
|
|
|
|
|
|
|
| 694 |
2. **Visualization Creation**:
|
| 695 |
-
|
| 696 |
-
|
|
|
|
| 697 |
3. **Performance Optimization**:
|
| 698 |
-
|
| 699 |
```python
|
| 700 |
if len(df) > 50000:
|
| 701 |
df = df.sample(5000, random_state=42)
|
| 702 |
```
|
|
|
|
| 703 |
4. **Layout and Styling**:
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
| 707 |
-
* Color schemes or color maps for visual elements.
|
| 708 |
-
* You must ensure that all axes (x and y) have **consistent formats** (e.g., using `K`, `M`, or 1,000 format, but not mixing formats).
|
| 709 |
5. **Trendlines**:
|
| 710 |
-
|
|
|
|
| 711 |
6. **Displaying the Visualization**:
|
| 712 |
* Use Plotly's `fig.show()` method to display the created chart.
|
| 713 |
-
|
|
|
|
| 714 |
7. **Error Handling**:
|
| 715 |
-
|
| 716 |
-
|
|
|
|
| 717 |
8. **No Data Modification**:
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 727 |
"""
|
| 728 |
goal = dspy.InputField(desc="User-defined chart goal (e.g. trendlines, scatter plots)")
|
| 729 |
dataset = dspy.InputField(desc="Details of the dataframe (`df`) and its columns")
|
| 730 |
styling_index = dspy.InputField(desc="Instructions for plot styling and layout formatting")
|
| 731 |
-
plan_instructions = dspy.InputField(desc="Variables to create and receive for visualization purposes")
|
| 732 |
|
| 733 |
code = dspy.OutputField(desc="Plotly Python code for the visualization")
|
| 734 |
summary = dspy.OutputField(desc="Plain-language summary of what is being visualized")
|
| 735 |
|
| 736 |
-
class
|
| 737 |
"""
|
| 738 |
-
|
| 739 |
-
You are a statistical analytics agent in a multi-agent data analytics pipeline.
|
| 740 |
You are given:
|
| 741 |
* A dataset (usually a cleaned or transformed version like `df_cleaned`).
|
| 742 |
* A user-defined goal (e.g., regression, seasonal decomposition).
|
| 743 |
-
*
|
| 744 |
-
* Which
|
| 745 |
-
* Which
|
| 746 |
-
* A set of
|
| 747 |
-
|
|
|
|
| 748 |
* Use the `statsmodels` library to implement the required statistical analysis.
|
| 749 |
* Ensure that all strings are handled as categorical variables via `C(col)` in model formulas.
|
| 750 |
* Always add a constant using `sm.add_constant()`.
|
| 751 |
-
* Do
|
| 752 |
* Convert `X` and `y` to float before fitting the model.
|
| 753 |
* Handle missing values before modeling.
|
| 754 |
* Avoid any data visualization (that is handled by another agent).
|
| 755 |
* Write output to the console using `print()`.
|
| 756 |
-
|
|
|
|
| 757 |
* Use `statsmodels.OLS` with proper handling of categorical variables and adding a constant term.
|
| 758 |
* Handle missing values appropriately.
|
| 759 |
-
|
|
|
|
| 760 |
* Use `statsmodels.tsa.seasonal_decompose`.
|
| 761 |
* Ensure the time series and period are correctly provided (i.e., `period` should not be `None`).
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
*
|
| 766 |
-
*
|
| 767 |
-
*
|
| 768 |
-
|
| 769 |
-
2.
|
| 770 |
-
|
| 771 |
-
4. **Do not reassign or modify** any variables passed via `plan_instructions`. These should be used as-is.
|
| 772 |
-
**Example Workflow:**
|
| 773 |
-
Given that the `plan_instructions` specifies variables to **CREATE** and **USE**, and includes instructions, your approach should look like this:
|
| 774 |
-
1. Use `df_cleaned` and the variables like `X` and `y` from `plan_instructions` for analysis.
|
| 775 |
-
2. Follow instructions for preprocessing (e.g., handle missing values or scale features).
|
| 776 |
-
3. If the goal is regression:
|
| 777 |
-
* Use `sm.OLS` for model fitting.
|
| 778 |
-
* Handle categorical variables via `C(col)` and add a constant term.
|
| 779 |
-
4. If the goal is seasonal decomposition:
|
| 780 |
-
* Ensure `period` is provided and use `sm.tsa.seasonal_decompose`.
|
| 781 |
-
5. Store the output variable as specified in `plan_instructions['CREATE']`.
|
| 782 |
### Example Code Structure:
|
| 783 |
```python
|
| 784 |
import statsmodels.api as sm
|
|
@@ -808,109 +919,89 @@ def statistical_model(X, y, goal, period=None):
|
|
| 808 |
except Exception as e:
|
| 809 |
return f"An error occurred: {e}"
|
| 810 |
```
|
| 811 |
-
|
| 812 |
-
|
| 813 |
-
|
| 814 |
-
|
|
|
|
| 815 |
4. Ensure reproducibility by setting the random state appropriately and handling categorical variables.
|
| 816 |
5. Focus on statistical analysis and avoid any unnecessary data manipulation.
|
| 817 |
-
|
| 818 |
-
|
| 819 |
-
*
|
|
|
|
| 820 |
* Respond in the user's language for all summary and reasoning but keep the code in english
|
| 821 |
"""
|
| 822 |
dataset = dspy.InputField(desc="Preprocessed dataset, often named df_cleaned")
|
| 823 |
goal = dspy.InputField(desc="The user's statistical analysis goal, e.g., regression or seasonal_decompose")
|
| 824 |
-
plan_instructions = dspy.InputField(desc="Instructions on variables to create and receive for statistical modeling")
|
| 825 |
|
| 826 |
code = dspy.OutputField(desc="Python code for statistical modeling using statsmodels")
|
| 827 |
summary = dspy.OutputField(desc="A concise bullet-point summary of the statistical analysis performed and key findings")
|
| 828 |
|
| 829 |
-
|
| 830 |
-
|
| 831 |
-
class planner_sk_learn_agent(dspy.Signature):
|
| 832 |
"""
|
| 833 |
-
|
| 834 |
-
You are a machine learning agent in a multi-agent data analytics pipeline.
|
| 835 |
You are given:
|
| 836 |
* A dataset (often cleaned and feature-engineered).
|
| 837 |
* A user-defined goal (e.g., classification, regression, clustering).
|
| 838 |
-
|
| 839 |
-
|
| 840 |
-
|
| 841 |
-
|
| 842 |
-
|
|
|
|
| 843 |
* Use the scikit-learn library to implement the appropriate ML pipeline.
|
| 844 |
* Always split data into training and testing sets where applicable.
|
| 845 |
* Use `print()` for all outputs.
|
| 846 |
* Ensure your code is:
|
| 847 |
-
|
| 848 |
-
|
| 849 |
-
|
| 850 |
* Your task may include:
|
| 851 |
* Preprocessing inputs (e.g., encoding).
|
| 852 |
* Model selection and training.
|
| 853 |
* Evaluation (e.g., accuracy, RMSE, classification report).
|
| 854 |
-
|
|
|
|
| 855 |
* Visualize anything (that's another agent's job).
|
| 856 |
-
|
| 857 |
-
|
| 858 |
-
|
| 859 |
-
|
| 860 |
-
|
| 861 |
-
|
| 862 |
-
|
| 863 |
-
|
| 864 |
-
|
| 865 |
-
|
| 866 |
-
|
| 867 |
-
|
| 868 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 869 |
3. If instructions are provided (e.g., scale or encode), follow them.
|
| 870 |
4. Split data into training and testing sets using `train_test_split`.
|
| 871 |
5. Train the model based on the received goal (classification, regression, etc.).
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
| 876 |
-
|
| 877 |
-
|
| 878 |
-
|
| 879 |
-
# Ensure that all variables follow plan instructions:
|
| 880 |
-
# Use received inputs: df_cleaned, feature_columns, target_column
|
| 881 |
-
X = df_cleaned[feature_columns]
|
| 882 |
-
y = df_cleaned[target_column]
|
| 883 |
-
# Apply any preprocessing instructions (e.g., scaling if instructed)
|
| 884 |
-
if 'scale' in plan_instructions['INSTRUCTIONS']:
|
| 885 |
-
scaler = StandardScaler()
|
| 886 |
-
X = scaler.fit_transform(X)
|
| 887 |
-
# Split the data into training and testing sets
|
| 888 |
-
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
|
| 889 |
-
# Select and train the model (based on the task)
|
| 890 |
-
model = LogisticRegression(random_state=42)
|
| 891 |
-
model.fit(X_train, y_train)
|
| 892 |
-
# Generate predictions
|
| 893 |
-
predictions = model.predict(X_test)
|
| 894 |
-
# Create the variable specified in 'plan_instructions': 'metrics'
|
| 895 |
-
metrics = classification_report(y_test, predictions)
|
| 896 |
-
# Print the results
|
| 897 |
-
print(metrics)
|
| 898 |
-
# Ensure the 'metrics' variable is returned as requested in the plan
|
| 899 |
-
```
|
| 900 |
-
**Summary:**
|
| 901 |
-
1. Always **USE** the variables passed in `plan_instructions['USE']` to build the pipeline.
|
| 902 |
-
2. Only **CREATE** the variables specified in `plan_instructions['CREATE']`. Do not create any additional variables.
|
| 903 |
-
3. Follow any **additional instructions** in `plan_instructions['INSTRUCTIONS']` (e.g., preprocessing steps).
|
| 904 |
-
4. Ensure reproducibility by setting `random_state=42` wherever necessary.
|
| 905 |
5. Focus on model building, evaluation, and saving the required outputs—avoid any unnecessary variables.
|
| 906 |
-
|
| 907 |
-
|
| 908 |
-
|
|
|
|
| 909 |
* Respond in the user's language for all summary and reasoning but keep the code in english
|
| 910 |
"""
|
| 911 |
dataset = dspy.InputField(desc="Input dataset, often cleaned and feature-selected (e.g., df_cleaned)")
|
| 912 |
goal = dspy.InputField(desc="The user's machine learning goal (e.g., classification or regression)")
|
| 913 |
-
plan_instructions = dspy.InputField(desc="Instructions indicating what to create and what variables to receive")
|
| 914 |
|
| 915 |
code = dspy.OutputField(desc="Scikit-learn based machine learning code")
|
| 916 |
summary = dspy.OutputField(desc="Explanation of the ML approach and evaluation")
|
|
@@ -924,140 +1015,7 @@ class goal_refiner_agent(dspy.Signature):
|
|
| 924 |
goal = dspy.InputField(desc="The user defined goal ")
|
| 925 |
refined_goal = dspy.OutputField(desc='Refined goal that helps the planner agent plan better')
|
| 926 |
|
| 927 |
-
class preprocessing_agent(dspy.Signature):
|
| 928 |
-
"""You are a AI data-preprocessing agent. Generate clean and efficient Python code using NumPy and Pandas to perform introductory data preprocessing on a pre-loaded DataFrame df, based on the user's analysis goals.
|
| 929 |
-
Preprocessing Requirements:
|
| 930 |
-
1. Identify Column Types
|
| 931 |
-
- Separate columns into numeric and categorical using:
|
| 932 |
-
categorical_columns = df.select_dtypes(include=[object, 'category']).columns.tolist()
|
| 933 |
-
numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
|
| 934 |
-
2. Handle Missing Values
|
| 935 |
-
- Numeric columns: Impute missing values using the mean of each column
|
| 936 |
-
- Categorical columns: Impute missing values using the mode of each column
|
| 937 |
-
3. Convert Date Strings to Datetime
|
| 938 |
-
- For any column suspected to represent dates (in string format), convert it to datetime using:
|
| 939 |
-
def safe_to_datetime(date):
|
| 940 |
-
try:
|
| 941 |
-
return pd.to_datetime(date, errors='coerce', cache=False)
|
| 942 |
-
except (ValueError, TypeError):
|
| 943 |
-
return pd.NaT
|
| 944 |
-
df['datetime_column'] = df['datetime_column'].apply(safe_to_datetime)
|
| 945 |
-
- Replace 'datetime_column' with the actual column names containing date-like strings
|
| 946 |
-
Important Notes:
|
| 947 |
-
- Do NOT create a correlation matrix — correlation analysis is outside the scope of preprocessing
|
| 948 |
-
- Do NOT generate any plots or visualizations
|
| 949 |
-
Output Instructions:
|
| 950 |
-
1. Include the full preprocessing Python code
|
| 951 |
-
2. Provide a brief bullet-point summary of the steps performed. Example:
|
| 952 |
-
• Identified 5 numeric and 4 categorical columns
|
| 953 |
-
• Filled missing numeric values with column means
|
| 954 |
-
• Filled missing categorical values with column modes
|
| 955 |
-
• Converted 1 date column to datetime format
|
| 956 |
-
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 957 |
-
"""
|
| 958 |
-
dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df, column_names set df as copy of df")
|
| 959 |
-
goal = dspy.InputField(desc="The user defined goal could ")
|
| 960 |
-
code = dspy.OutputField(desc ="The code that does the data preprocessing and introductory analysis")
|
| 961 |
-
summary = dspy.OutputField(desc="A concise bullet-point summary of the preprocessing operations performed")
|
| 962 |
-
|
| 963 |
-
|
| 964 |
-
|
| 965 |
-
class statistical_analytics_agent(dspy.Signature):
|
| 966 |
-
# Statistical Analysis Agent, builds statistical models using StatsModel Package
|
| 967 |
-
"""
|
| 968 |
-
You are a statistical analytics agent. Your task is to take a dataset and a user-defined goal and output Python code that performs the appropriate statistical analysis to achieve that goal. Follow these guidelines:
|
| 969 |
-
IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
|
| 970 |
-
Data Handling:
|
| 971 |
-
Always handle strings as categorical variables in a regression using statsmodels C(string_column).
|
| 972 |
-
Do not change the index of the DataFrame.
|
| 973 |
-
Convert X and y into float when fitting a model.
|
| 974 |
-
Error Handling:
|
| 975 |
-
Always check for missing values and handle them appropriately.
|
| 976 |
-
Ensure that categorical variables are correctly processed.
|
| 977 |
-
Provide clear error messages if the model fitting fails.
|
| 978 |
-
Regression:
|
| 979 |
-
For regression, use statsmodels and ensure that a constant term is added to the predictor using sm.add_constant(X).
|
| 980 |
-
Handle categorical variables using C(column_name) in the model formula.
|
| 981 |
-
Fit the model with model = sm.OLS(y.astype(float), X.astype(float)).fit().
|
| 982 |
-
Seasonal Decomposition:
|
| 983 |
-
Ensure the period is set correctly when performing seasonal decomposition.
|
| 984 |
-
Verify the number of observations works for the decomposition.
|
| 985 |
-
Output:
|
| 986 |
-
Ensure the code is executable and as intended.
|
| 987 |
-
Also choose the correct type of model for the problem
|
| 988 |
-
Avoid adding data visualization code.
|
| 989 |
-
Use code like this to prevent failing:
|
| 990 |
-
import pandas as pd
|
| 991 |
-
import numpy as np
|
| 992 |
-
import statsmodels.api as sm
|
| 993 |
-
def statistical_model(X, y, goal, period=None):
|
| 994 |
-
try:
|
| 995 |
-
# Check for missing values and handle them
|
| 996 |
-
X = X.dropna()
|
| 997 |
-
y = y.loc[X.index].dropna()
|
| 998 |
-
# Ensure X and y are aligned
|
| 999 |
-
X = X.loc[y.index]
|
| 1000 |
-
# Convert categorical variables
|
| 1001 |
-
for col in X.select_dtypes(include=['object', 'category']).columns:
|
| 1002 |
-
X[col] = X[col].astype('category')
|
| 1003 |
-
# Add a constant term to the predictor
|
| 1004 |
-
X = sm.add_constant(X)
|
| 1005 |
-
# Fit the model
|
| 1006 |
-
if goal == 'regression':
|
| 1007 |
-
# Handle categorical variables in the model formula
|
| 1008 |
-
formula = 'y ~ ' + ' + '.join([f'C({col})' if X[col].dtype.name == 'category' else col for col in X.columns])
|
| 1009 |
-
model = sm.OLS(y.astype(float), X.astype(float)).fit()
|
| 1010 |
-
return model.summary()
|
| 1011 |
-
elif goal == 'seasonal_decompose':
|
| 1012 |
-
if period is None:
|
| 1013 |
-
raise ValueError("Period must be specified for seasonal decomposition")
|
| 1014 |
-
decomposition = sm.tsa.seasonal_decompose(y, period=period)
|
| 1015 |
-
return decomposition
|
| 1016 |
-
else:
|
| 1017 |
-
raise ValueError("Unknown goal specified. Please provide a valid goal.")
|
| 1018 |
-
except Exception as e:
|
| 1019 |
-
return f"An error occurred: {e}"
|
| 1020 |
-
# Example usage:
|
| 1021 |
-
result = statistical_analysis(X, y, goal='regression')
|
| 1022 |
-
print(result)
|
| 1023 |
-
If visualizing use plotly
|
| 1024 |
-
Provide a concise bullet-point summary of the statistical analysis performed.
|
| 1025 |
-
|
| 1026 |
-
Example Summary:
|
| 1027 |
-
• Applied linear regression with OLS to predict house prices based on 5 features
|
| 1028 |
-
• Model achieved R-squared of 0.78
|
| 1029 |
-
• Significant predictors include square footage (p<0.001) and number of bathrooms (p<0.01)
|
| 1030 |
-
• Detected strong seasonal pattern with 12-month periodicity
|
| 1031 |
-
• Forecast shows 15% growth trend over next quarter
|
| 1032 |
-
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 1033 |
-
"""
|
| 1034 |
-
dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df,columns set df as copy of df")
|
| 1035 |
-
goal = dspy.InputField(desc="The user defined goal for the analysis to be performed")
|
| 1036 |
-
code = dspy.OutputField(desc ="The code that does the statistical analysis using statsmodel")
|
| 1037 |
-
summary = dspy.OutputField(desc="A concise bullet-point summary of the statistical analysis performed and key findings")
|
| 1038 |
-
|
| 1039 |
|
| 1040 |
-
class sk_learn_agent(dspy.Signature):
|
| 1041 |
-
# Machine Learning Agent, performs task using sci-kit learn
|
| 1042 |
-
"""You are a machine learning agent.
|
| 1043 |
-
Your task is to take a dataset and a user-defined goal, and output Python code that performs the appropriate machine learning analysis to achieve that goal.
|
| 1044 |
-
You should use the scikit-learn library.
|
| 1045 |
-
IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
|
| 1046 |
-
Make sure your output is as intended!
|
| 1047 |
-
Provide a concise bullet-point summary of the machine learning operations performed.
|
| 1048 |
-
|
| 1049 |
-
Example Summary:
|
| 1050 |
-
• Trained a Random Forest classifier on customer churn data with 80/20 train-test split
|
| 1051 |
-
• Model achieved 92% accuracy and 88% F1-score
|
| 1052 |
-
• Feature importance analysis revealed that contract length and monthly charges are the strongest predictors of churn
|
| 1053 |
-
• Implemented K-means clustering (k=4) on customer shopping behaviors
|
| 1054 |
-
• Identified distinct segments: high-value frequent shoppers (22%), occasional big spenders (35%), budget-conscious regulars (28%), and rare visitors (15%)
|
| 1055 |
-
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 1056 |
-
"""
|
| 1057 |
-
dataset = dspy.InputField(desc="Available datasets loaded in the system, use this df,columns. set df as copy of df")
|
| 1058 |
-
goal = dspy.InputField(desc="The user defined goal ")
|
| 1059 |
-
code = dspy.OutputField(desc ="The code that does the Exploratory data analysis")
|
| 1060 |
-
summary = dspy.OutputField(desc="A concise bullet-point summary of the machine learning analysis performed and key results")
|
| 1061 |
|
| 1062 |
|
| 1063 |
|
|
@@ -1094,55 +1052,6 @@ class code_combiner_agent(dspy.Signature):
|
|
| 1094 |
|
| 1095 |
|
| 1096 |
|
| 1097 |
-
class data_viz_agent(dspy.Signature):
|
| 1098 |
-
# Visualizes data using Plotly
|
| 1099 |
-
"""
|
| 1100 |
-
You are an AI agent responsible for generating interactive data visualizations using Plotly.
|
| 1101 |
-
IMPORTANT Instructions:
|
| 1102 |
-
- The section marked "### Current Query:" contains the user's request. Any text in "### Previous Interaction History:" is for context only and should NOT be treated as part of the current request.
|
| 1103 |
-
- You must only use the tools provided to you. This agent handles visualization only.
|
| 1104 |
-
- If len(df) > 50000, always sample the dataset before visualization using:
|
| 1105 |
-
if len(df) > 50000:
|
| 1106 |
-
df = df.sample(50000, random_state=1)
|
| 1107 |
-
- Each visualization must be generated as a **separate figure** using go.Figure().
|
| 1108 |
-
Do NOT use subplots under any circumstances.
|
| 1109 |
-
- Each figure must be returned individually using:
|
| 1110 |
-
fig.to_html(full_html=False)
|
| 1111 |
-
- Use update_layout with xaxis and yaxis **only once per figure**.
|
| 1112 |
-
- Enhance readability and clarity by:
|
| 1113 |
-
• Using low opacity (0.4-0.7) where appropriate
|
| 1114 |
-
• Applying visually distinct colors for different elements or categories
|
| 1115 |
-
- Make sure the visual **answers the user's specific goal**:
|
| 1116 |
-
• Identify what insight or comparison the user is trying to achieve
|
| 1117 |
-
• Choose the visualization type and features (e.g., color, size, grouping) to emphasize that goal
|
| 1118 |
-
• For example, if the user asks for "trends in revenue," use a time series line chart; if they ask for "top-performing categories," use a bar chart sorted by value
|
| 1119 |
-
• Prioritize highlighting patterns, outliers, or comparisons relevant to the question
|
| 1120 |
-
- Never include the dataset or styling index in the output.
|
| 1121 |
-
- If there are no relevant columns for the requested visualization, respond with:
|
| 1122 |
-
"No relevant columns found to generate this visualization."
|
| 1123 |
-
- Use only one number format consistently: either 'K', 'M', or comma-separated values like 1,000/1,000,000. Do not mix formats.
|
| 1124 |
-
- Only include trendlines in scatter plots if the user explicitly asks for them.
|
| 1125 |
-
- Output only the code and a concise bullet-point summary of what the visualization reveals.
|
| 1126 |
-
- Always end each visualization with:
|
| 1127 |
-
fig.to_html(full_html=False)
|
| 1128 |
-
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 1129 |
-
Example Summary:
|
| 1130 |
-
• Created an interactive scatter plot of sales vs. marketing spend with color-coded product categories
|
| 1131 |
-
• Included a trend line showing positive correlation (r=0.72)
|
| 1132 |
-
• Highlighted outliers where high marketing spend resulted in low sales
|
| 1133 |
-
• Generated a time series chart of monthly revenue from 2020-2023
|
| 1134 |
-
• Added annotations for key business events
|
| 1135 |
-
• Visualization reveals 35% YoY growth with seasonal peaks in Q4
|
| 1136 |
-
|
| 1137 |
-
"""
|
| 1138 |
-
goal = dspy.InputField(desc="user defined goal which includes information about data and chart they want to plot")
|
| 1139 |
-
dataset = dspy.InputField(desc=" Provides information about the data in the data frame. Only use column names and dataframe_name as in this context")
|
| 1140 |
-
styling_index = dspy.InputField(desc='Provides instructions on how to style your Plotly plots')
|
| 1141 |
-
code= dspy.OutputField(desc="Plotly code that visualizes what the user needs according to the query & dataframe_index & styling_context")
|
| 1142 |
-
summary = dspy.OutputField(desc="A concise bullet-point summary of the visualization created and key insights revealed")
|
| 1143 |
-
|
| 1144 |
-
|
| 1145 |
-
|
| 1146 |
class code_fix(dspy.Signature):
|
| 1147 |
"""
|
| 1148 |
You are an expert AI developer and data analyst assistant, skilled at identifying and resolving issues in Python code related to data analytics. Another agent has attempted to generate Python code for a data analytics task but produced code that is broken or throws an error.
|
|
@@ -1218,42 +1127,89 @@ class auto_analyst_ind(dspy.Module):
|
|
| 1218 |
self.agent_inputs = {}
|
| 1219 |
self.agent_desc = []
|
| 1220 |
|
| 1221 |
-
#
|
| 1222 |
-
|
| 1223 |
-
|
| 1224 |
-
|
| 1225 |
-
|
| 1226 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1227 |
|
| 1228 |
-
# Load
|
| 1229 |
-
if db_session:
|
| 1230 |
try:
|
| 1231 |
-
|
|
|
|
| 1232 |
|
| 1233 |
for template_name, signature in template_signatures.items():
|
| 1234 |
# Add template agent to agents dict
|
| 1235 |
-
self.agents[template_name] = dspy.asyncify(dspy.
|
| 1236 |
|
| 1237 |
-
# Extract input fields from signature - templates use standard fields
|
| 1238 |
-
self.agent_inputs[template_name] = {'goal', 'dataset', 'styling_index'
|
| 1239 |
|
| 1240 |
-
#
|
| 1241 |
-
|
|
|
|
| 1242 |
|
| 1243 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1244 |
|
| 1245 |
except Exception as e:
|
| 1246 |
-
logger.log_message(f"Error loading
|
| 1247 |
-
|
| 1248 |
-
|
| 1249 |
-
|
|
|
|
|
|
|
|
|
|
| 1250 |
self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
|
| 1251 |
self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
|
| 1252 |
-
# self.code_combiner_agent = dspy.ChainOfThought(code_combiner_agent)
|
| 1253 |
|
| 1254 |
# Store user_id for usage tracking
|
| 1255 |
self.user_id = user_id
|
| 1256 |
-
|
| 1257 |
async def _track_agent_usage(self, agent_name):
|
| 1258 |
"""Track usage for template agents"""
|
| 1259 |
try:
|
|
@@ -1422,66 +1378,110 @@ class auto_analyst(dspy.Module):
|
|
| 1422 |
self.agent_inputs = {}
|
| 1423 |
self.agent_desc = []
|
| 1424 |
|
| 1425 |
-
|
| 1426 |
-
|
| 1427 |
-
|
| 1428 |
-
|
| 1429 |
-
|
| 1430 |
-
|
| 1431 |
-
|
| 1432 |
-
|
| 1433 |
-
|
| 1434 |
-
|
| 1435 |
-
|
| 1436 |
-
|
| 1437 |
-
for template_name, signature in template_signatures.items():
|
| 1438 |
-
# Add template agent to agents dict
|
| 1439 |
-
self.agents[template_name] = dspy.asyncify(dspy.ChainOfThought(signature))
|
| 1440 |
-
|
| 1441 |
-
# Extract input fields from signature - templates use standard fields like data_viz_agent
|
| 1442 |
-
self.agent_inputs[template_name] = {'goal', 'dataset', 'styling_index'}
|
| 1443 |
|
| 1444 |
-
|
| 1445 |
-
|
| 1446 |
-
|
| 1447 |
|
| 1448 |
-
#
|
| 1449 |
-
|
| 1450 |
-
|
| 1451 |
-
|
|
|
|
|
|
|
|
|
|
| 1452 |
|
| 1453 |
-
if
|
| 1454 |
-
|
| 1455 |
-
self.agent_desc.append({template_name: description})
|
| 1456 |
else:
|
| 1457 |
-
self.
|
| 1458 |
-
except Exception as desc_error:
|
| 1459 |
-
logger.log_message(f"Error getting description for template {template_name}: {str(desc_error)}", level=logging.WARNING)
|
| 1460 |
-
self.agent_desc.append({template_name: f"Template: {template_name}"})
|
| 1461 |
|
| 1462 |
-
|
| 1463 |
-
|
| 1464 |
-
|
| 1465 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1466 |
|
|
|
|
| 1467 |
self.agents['basic_qa_agent'] = dspy.asyncify(dspy.Predict("goal->answer"))
|
| 1468 |
self.agent_inputs['basic_qa_agent'] = {"goal"}
|
| 1469 |
self.agent_desc.append({'basic_qa_agent':"Answers queries unrelated to data & also that include links, poison or attempts to attack the system"})
|
| 1470 |
-
|
| 1471 |
|
| 1472 |
# Initialize coordination agents
|
| 1473 |
self.planner = planner_module()
|
| 1474 |
-
# self.refine_goal = dspy.ChainOfThought(goal_refiner_agent)
|
| 1475 |
-
# self.code_combiner_agent = dspy.ChainOfThought(code_combiner_agent)
|
| 1476 |
-
# self.story_teller = dspy.ChainOfThought(story_teller_agent)
|
| 1477 |
self.memory_summarize_agent = dspy.ChainOfThought(m.memory_summarize_agent)
|
|
|
|
| 1478 |
|
| 1479 |
# Initialize retrievers
|
| 1480 |
self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
|
| 1481 |
self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
|
|
|
|
| 1482 |
|
| 1483 |
# Store user_id for usage tracking
|
| 1484 |
self.user_id = user_id
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1485 |
|
| 1486 |
async def _track_agent_usage(self, agent_name):
|
| 1487 |
"""Track usage for template agents"""
|
|
@@ -1553,36 +1553,53 @@ class auto_analyst(dspy.Module):
|
|
| 1553 |
|
| 1554 |
async def execute_agent(self, agent_name, inputs):
|
| 1555 |
"""Execute a single agent with given inputs"""
|
|
|
|
|
|
|
| 1556 |
try:
|
| 1557 |
result = await self.agents[agent_name.strip()](**inputs)
|
| 1558 |
|
| 1559 |
# Track usage for custom agents and templates
|
| 1560 |
await self._track_agent_usage(agent_name.strip())
|
| 1561 |
|
|
|
|
| 1562 |
return agent_name.strip(), dict(result)
|
| 1563 |
except Exception as e:
|
|
|
|
| 1564 |
return agent_name.strip(), {"error": str(e)}
|
| 1565 |
|
| 1566 |
async def get_plan(self, query):
|
| 1567 |
"""Get the analysis plan"""
|
|
|
|
|
|
|
| 1568 |
dict_ = {}
|
| 1569 |
dict_['dataset'] = self.dataset.retrieve(query)[0].text
|
| 1570 |
dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
|
| 1571 |
dict_['goal'] = query
|
| 1572 |
dict_['Agent_desc'] = str(self.agent_desc)
|
| 1573 |
|
| 1574 |
-
|
| 1575 |
-
|
| 1576 |
-
|
| 1577 |
-
|
| 1578 |
-
|
| 1579 |
-
|
| 1580 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1581 |
|
| 1582 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1583 |
|
| 1584 |
async def execute_plan(self, query, plan):
|
| 1585 |
"""Execute the plan and yield results as they complete"""
|
|
|
|
|
|
|
| 1586 |
dict_ = {}
|
| 1587 |
dict_['dataset'] = self.dataset.retrieve(query)[0].text
|
| 1588 |
dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
|
|
@@ -1593,91 +1610,67 @@ class auto_analyst(dspy.Module):
|
|
| 1593 |
|
| 1594 |
# Clean and split the plan string into agent names
|
| 1595 |
plan_text = plan.get("plan", "").replace("Plan", "").replace(":", "").strip()
|
| 1596 |
-
|
| 1597 |
|
| 1598 |
if "basic_qa_agent" in plan_text:
|
|
|
|
| 1599 |
inputs = dict(goal=query)
|
| 1600 |
-
agent_name, response = await self.execute_agent('basic_qa_agent', inputs)
|
| 1601 |
yield agent_name, inputs, response
|
| 1602 |
return
|
| 1603 |
|
| 1604 |
plan_list = [agent.strip() for agent in plan_text.split("->") if agent.strip()]
|
|
|
|
| 1605 |
|
| 1606 |
# Parse the attached plan_instructions into a dict
|
| 1607 |
raw_instr = plan.get("plan_instructions", {})
|
| 1608 |
if isinstance(raw_instr, str):
|
| 1609 |
try:
|
| 1610 |
plan_instructions = json.loads(raw_instr)
|
| 1611 |
-
except Exception:
|
|
|
|
| 1612 |
plan_instructions = {}
|
| 1613 |
elif isinstance(raw_instr, dict):
|
| 1614 |
-
plan_instructions =
|
| 1615 |
else:
|
| 1616 |
plan_instructions = {}
|
| 1617 |
|
| 1618 |
-
|
| 1619 |
-
if not plan_list:
|
| 1620 |
-
yield "plan_not_found", dict(plan), {"error": "No plan found"}
|
| 1621 |
-
return
|
| 1622 |
-
|
| 1623 |
|
| 1624 |
-
#
|
| 1625 |
-
|
| 1626 |
-
|
|
|
|
|
|
|
| 1627 |
|
| 1628 |
-
|
| 1629 |
-
|
| 1630 |
-
|
| 1631 |
-
|
| 1632 |
-
|
| 1633 |
-
|
| 1634 |
-
|
| 1635 |
-
}
|
| 1636 |
|
| 1637 |
-
|
| 1638 |
-
|
| 1639 |
-
|
| 1640 |
-
current_instructions = plan_instructions.get(key, {"create": [], "use": [], "instruction": ""})
|
| 1641 |
|
| 1642 |
-
#
|
| 1643 |
-
|
|
|
|
|
|
|
|
|
|
| 1644 |
|
| 1645 |
-
|
| 1646 |
-
if idx > 0:
|
| 1647 |
-
prev_agent = plan_list[idx-1].strip()
|
| 1648 |
-
prev_instructions = plan_instructions.get(prev_agent, {}).get("instruction", "")
|
| 1649 |
-
formatted_instructions[f"Previous Agent {prev_agent}"] = prev_instructions
|
| 1650 |
|
| 1651 |
-
#
|
| 1652 |
-
|
| 1653 |
-
|
| 1654 |
-
next_instructions = plan_instructions.get(next_agent, {}).get("instruction", "")
|
| 1655 |
-
formatted_instructions[f"Next Agent {next_agent}"] = next_instructions
|
| 1656 |
|
| 1657 |
-
|
| 1658 |
-
|
| 1659 |
-
|
| 1660 |
-
# Create async task directly from the asyncified agent
|
| 1661 |
-
task = self.execute_agent(agent_name, inputs)
|
| 1662 |
-
tasks.append(task)
|
| 1663 |
-
task_info.append((agent_name, inputs))
|
| 1664 |
-
|
| 1665 |
-
# Execute all tasks concurrently and yield results as they complete
|
| 1666 |
-
try:
|
| 1667 |
-
# Execute all tasks concurrently
|
| 1668 |
-
results = await asyncio.gather(*tasks, return_exceptions=True)
|
| 1669 |
-
|
| 1670 |
-
# Yield results with their corresponding task info
|
| 1671 |
-
for i, result in enumerate(results):
|
| 1672 |
-
agent_name, inputs = task_info[i]
|
| 1673 |
-
|
| 1674 |
-
if isinstance(result, Exception):
|
| 1675 |
-
yield agent_name, inputs, {"error": str(result)}
|
| 1676 |
-
else:
|
| 1677 |
-
name, response = result
|
| 1678 |
-
yield name, inputs, response
|
| 1679 |
|
| 1680 |
-
|
| 1681 |
-
|
| 1682 |
-
|
| 1683 |
|
|
|
|
| 22 |
A dspy.Signature class with the custom prompt and standard input/output fields
|
| 23 |
"""
|
| 24 |
|
| 25 |
+
# Check if this is a visualization agent to determine input fields
|
| 26 |
+
is_viz_agent = 'viz' in agent_name.lower() or 'visual' in agent_name.lower() or 'plot' in agent_name.lower() or 'chart' in agent_name.lower()
|
| 27 |
+
|
| 28 |
+
# Standard input/output fields that match the unified agent signatures
|
| 29 |
class_attributes = {
|
| 30 |
'__doc__': prompt_template, # The custom prompt becomes the docstring
|
| 31 |
'goal': dspy.InputField(desc="User-defined goal which includes information about data and task they want to perform"),
|
| 32 |
'dataset': dspy.InputField(desc="Provides information about the data in the data frame. Only use column names and dataframe_name as in this context"),
|
| 33 |
+
'plan_instructions': dspy.InputField(desc="Agent-level instructions about what to create and receive (optional for individual use)", default=""),
|
| 34 |
'code': dspy.OutputField(desc="Generated Python code for the analysis"),
|
| 35 |
'summary': dspy.OutputField(desc="A concise bullet-point summary of what was done and key results")
|
| 36 |
}
|
| 37 |
|
| 38 |
+
# Add styling_index for visualization agents
|
| 39 |
+
if is_viz_agent:
|
| 40 |
+
class_attributes['styling_index'] = dspy.InputField(desc='Provides instructions on how to style outputs and formatting')
|
| 41 |
+
|
| 42 |
# Create the dynamic signature class
|
| 43 |
CustomAgentSignature = type(agent_name, (dspy.Signature,), class_attributes)
|
| 44 |
return CustomAgentSignature
|
|
|
|
| 46 |
def load_user_enabled_templates_from_db(user_id, db_session):
|
| 47 |
"""
|
| 48 |
Load template agents that are enabled for a specific user from the database.
|
| 49 |
+
Default agents are enabled by default unless explicitly disabled by user preference.
|
| 50 |
|
| 51 |
Args:
|
| 52 |
user_id: ID of the user
|
|
|
|
| 63 |
if not user_id:
|
| 64 |
return agent_signatures
|
| 65 |
|
| 66 |
+
# Get list of default agent names that should be enabled by default
|
| 67 |
+
default_agent_names = [
|
| 68 |
+
"preprocessing_agent",
|
| 69 |
+
"statistical_analytics_agent",
|
| 70 |
+
"sk_learn_agent",
|
| 71 |
+
"data_viz_agent"
|
| 72 |
+
]
|
| 73 |
+
|
| 74 |
# Get all active templates
|
| 75 |
all_templates = db_session.query(AgentTemplate).filter(
|
| 76 |
AgentTemplate.is_active == True
|
|
|
|
| 83 |
UserTemplatePreference.template_id == template.template_id
|
| 84 |
).first()
|
| 85 |
|
| 86 |
+
# Determine if template should be enabled by default
|
| 87 |
+
is_default_agent = template.template_name in default_agent_names
|
| 88 |
+
default_enabled = is_default_agent # Default agents enabled by default, others disabled
|
| 89 |
|
| 90 |
+
# Template is enabled by default for default agents, disabled for others
|
| 91 |
+
is_enabled = preference.is_enabled if preference else default_enabled
|
| 92 |
+
|
| 93 |
if is_enabled:
|
| 94 |
# Create dynamic signature for each enabled template
|
| 95 |
signature = create_custom_agent_signature(
|
|
|
|
| 108 |
def load_user_enabled_templates_for_planner_from_db(user_id, db_session):
|
| 109 |
"""
|
| 110 |
Load template agents that are enabled for planner use (max 10, prioritized by usage).
|
| 111 |
+
Default agents are enabled by default unless explicitly disabled by user preference.
|
| 112 |
|
| 113 |
Args:
|
| 114 |
user_id: ID of the user
|
|
|
|
| 119 |
"""
|
| 120 |
try:
|
| 121 |
from src.db.schemas.models import AgentTemplate, UserTemplatePreference
|
| 122 |
+
from datetime import datetime, UTC
|
| 123 |
|
| 124 |
agent_signatures = {}
|
| 125 |
|
| 126 |
if not user_id:
|
| 127 |
return agent_signatures
|
| 128 |
|
| 129 |
+
# Get list of default agent names that should be enabled by default
|
| 130 |
+
default_agent_names = [
|
| 131 |
+
"preprocessing_agent",
|
| 132 |
+
"statistical_analytics_agent",
|
| 133 |
+
"sk_learn_agent",
|
| 134 |
+
"data_viz_agent"
|
| 135 |
+
]
|
|
|
|
| 136 |
|
| 137 |
+
# Get all active templates
|
| 138 |
+
all_templates = db_session.query(AgentTemplate).filter(
|
| 139 |
+
AgentTemplate.is_active == True
|
| 140 |
+
).all()
|
| 141 |
+
|
| 142 |
+
enabled_templates = []
|
| 143 |
+
for template in all_templates:
|
| 144 |
+
# Check if user has a preference record for this template
|
| 145 |
+
preference = db_session.query(UserTemplatePreference).filter(
|
| 146 |
+
UserTemplatePreference.user_id == user_id,
|
| 147 |
+
UserTemplatePreference.template_id == template.template_id
|
| 148 |
).first()
|
| 149 |
|
| 150 |
+
# Determine if template should be enabled by default
|
| 151 |
+
is_default_agent = template.template_name in default_agent_names
|
| 152 |
+
default_enabled = is_default_agent # Default agents enabled by default, others disabled
|
| 153 |
+
|
| 154 |
+
# Template is enabled by default for default agents, disabled for others
|
| 155 |
+
is_enabled = preference.is_enabled if preference else default_enabled
|
| 156 |
+
|
| 157 |
+
if is_enabled:
|
| 158 |
+
enabled_templates.append({
|
| 159 |
+
'template': template,
|
| 160 |
+
'preference': preference,
|
| 161 |
+
'usage_count': preference.usage_count if preference else 0,
|
| 162 |
+
'last_used_at': preference.last_used_at if preference else None
|
| 163 |
+
})
|
| 164 |
+
|
| 165 |
+
# Sort by usage (most used first) and limit to 10
|
| 166 |
+
enabled_templates.sort(key=lambda x: (x['usage_count'], x['last_used_at'] or datetime.min.replace(tzinfo=UTC)), reverse=True)
|
| 167 |
+
enabled_templates = enabled_templates[:10]
|
| 168 |
+
|
| 169 |
+
for item in enabled_templates:
|
| 170 |
+
template = item['template']
|
| 171 |
+
# Create dynamic signature for each enabled template
|
| 172 |
+
signature = create_custom_agent_signature(
|
| 173 |
+
template.template_name,
|
| 174 |
+
template.description,
|
| 175 |
+
template.prompt_template
|
| 176 |
+
)
|
| 177 |
+
agent_signatures[template.template_name] = signature
|
| 178 |
|
| 179 |
logger.log_message(f"Loaded {len(agent_signatures)} templates for planner", level=logging.DEBUG)
|
| 180 |
return agent_signatures
|
|
|
|
| 296 |
)
|
| 297 |
agent_signatures[template.template_name] = signature
|
| 298 |
|
|
|
|
| 299 |
return agent_signatures
|
| 300 |
|
| 301 |
except Exception as e:
|
|
|
|
| 306 |
|
| 307 |
# === END CUSTOM AGENT FUNCTIONALITY ===
|
| 308 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
def get_agent_description(agent_name, is_planner=False):
|
| 310 |
+
"""
|
| 311 |
+
Get agent description from database instead of hardcoded dictionaries.
|
| 312 |
+
This function is kept for backward compatibility but will fetch from DB.
|
| 313 |
+
"""
|
| 314 |
+
try:
|
| 315 |
+
from src.db.init_db import session_factory
|
| 316 |
+
from src.db.schemas.models import AgentTemplate
|
| 317 |
+
|
| 318 |
+
db_session = session_factory()
|
| 319 |
+
try:
|
| 320 |
+
template = db_session.query(AgentTemplate).filter(
|
| 321 |
+
AgentTemplate.template_name == agent_name,
|
| 322 |
+
AgentTemplate.is_active == True
|
| 323 |
+
).first()
|
| 324 |
+
|
| 325 |
+
if template:
|
| 326 |
+
return template.description
|
| 327 |
+
else:
|
| 328 |
+
return "No description available for this agent"
|
| 329 |
+
finally:
|
| 330 |
+
db_session.close()
|
| 331 |
+
except Exception as e:
|
| 332 |
+
return "No description available for this agent"
|
| 333 |
|
| 334 |
|
| 335 |
# Agent to make a Chat history name from a query
|
|
|
|
| 452 |
class advanced_query_planner(dspy.Signature):
|
| 453 |
"""
|
| 454 |
You are a advanced data analytics planner agent. Your task is to generate the most efficient plan—using the fewest necessary agents and variables—to achieve a user-defined goal. The plan must preserve data integrity, avoid unnecessary steps, and ensure clear data flow between agents.
|
| 455 |
+
|
| 456 |
+
**CRITICAL**: Before planning, check if any agents are available in Agent_desc. If Agent_desc is empty or contains no active agents, respond with:
|
| 457 |
+
plan: no_agents_available
|
| 458 |
+
plan_instructions: {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
|
| 459 |
+
|
| 460 |
**Inputs**:
|
| 461 |
1. Datasets (raw or preprocessed)
|
| 462 |
2. Agent descriptions (roles, variables they create/use, constraints)
|
|
|
|
| 473 |
Example: 1 agent use
|
| 474 |
goal: "Generate a bar plot showing sales by category after cleaning the raw data and calculating the average of the 'sales' column"
|
| 475 |
Output:
|
| 476 |
+
plan: data_viz_agent
|
| 477 |
{
|
| 478 |
+
"data_viz_agent": {
|
| 479 |
"create": [
|
| 480 |
"cleaned_data: DataFrame - cleaned version of df (pd.Dataframe) after removing null values"
|
| 481 |
],
|
|
|
|
| 487 |
}
|
| 488 |
Example 3 Agent
|
| 489 |
goal:"Clean the dataset, run a linear regression to model the relationship between marketing budget and sales, and visualize the regression line with confidence intervals."
|
| 490 |
+
plan: preprocessing_agent -> statistical_analytics_agent -> data_viz_agent
|
| 491 |
{
|
| 492 |
+
"preprocessing_agent": {
|
| 493 |
"create": [
|
| 494 |
"cleaned_data: DataFrame - cleaned version of df with missing values handled and proper data types inferred"
|
| 495 |
],
|
|
|
|
| 498 |
],
|
| 499 |
"instruction": "Clean df by handling missing values and converting column types (e.g., dates). Output cleaned_data for modeling."
|
| 500 |
},
|
| 501 |
+
"statistical_analytics_agent": {
|
| 502 |
"create": [
|
| 503 |
"regression_results: dict - model summary including coefficients, p-values, R², and confidence intervals"
|
| 504 |
],
|
|
|
|
| 507 |
],
|
| 508 |
"instruction": "Perform linear regression using cleaned_data to model sales as a function of marketing budget. Return regression_results including coefficients and confidence intervals."
|
| 509 |
},
|
| 510 |
+
"data_viz_agent": {
|
| 511 |
"create": [
|
| 512 |
"regression_plot: PlotlyFigure - visual plot showing regression line with confidence intervals"
|
| 513 |
],
|
|
|
|
| 532 |
"""
|
| 533 |
You are the basic query planner in the system, you pick one agent, to answer the user's goal.
|
| 534 |
Use the Agent_desc that describes the names and actions of agents available.
|
| 535 |
+
|
| 536 |
+
**CRITICAL**: Before planning, check if any agents are available in Agent_desc. If Agent_desc is empty or contains no active agents, respond with:
|
| 537 |
+
plan: no_agents_available
|
| 538 |
+
plan_instructions: {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
|
| 539 |
+
|
| 540 |
Example: Visualize height and salary?
|
| 541 |
+
plan:data_viz_agent
|
| 542 |
plan_instructions:
|
| 543 |
{
|
| 544 |
+
"data_viz_agent": {
|
| 545 |
"create": ["scatter_plot"],
|
| 546 |
"use": ["original_data"],
|
| 547 |
"instruction": "use the original_data to create scatter_plot of height & salary, using plotly"
|
| 548 |
}
|
| 549 |
}
|
| 550 |
Example: Tell me the correlation between X and Y
|
| 551 |
+
plan:preprocessing_agent
|
| 552 |
plan_instructions:{
|
| 553 |
+
"data_viz_agent": {
|
| 554 |
"create": ["correlation"],
|
| 555 |
"use": ["original_data"],
|
| 556 |
"instruction": "use the original_data to measure correlation of X & Y, using pandas"
|
|
|
|
| 576 |
3. User-defined Goal
|
| 577 |
You take these three inputs to develop a comprehensive plan to achieve the user-defined goal from the data & Agents available.
|
| 578 |
In case you think the user-defined goal is infeasible you can ask the user to redefine or add more description to the goal.
|
| 579 |
+
|
| 580 |
+
**CRITICAL**: Before planning, check if any agents are available in Agent_desc. If Agent_desc is empty or contains no active agents, respond with:
|
| 581 |
+
plan: no_agents_available
|
| 582 |
+
plan_instructions: {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
|
| 583 |
+
|
| 584 |
Give your output in this format:
|
| 585 |
plan: Agent1->Agent2
|
| 586 |
plan_instructions = {
|
|
|
|
| 629 |
|
| 630 |
self.allocator = dspy.Predict("goal,planner_desc,dataset->exact_word_complexity,reasoning")
|
| 631 |
|
| 632 |
+
async def forward(self, goal, dataset, Agent_desc):
|
| 633 |
+
logger.log_message(f"Planner forward called with goal: {goal[:100]}...", level=logging.INFO)
|
| 634 |
+
logger.log_message(f"Agent descriptions: {Agent_desc}", level=logging.DEBUG)
|
| 635 |
+
|
| 636 |
+
# Check if we have any agents available
|
| 637 |
+
if not Agent_desc or Agent_desc == "[]" or len(str(Agent_desc).strip()) < 10:
|
| 638 |
+
logger.log_message("No agents available for planning", level=logging.WARNING)
|
| 639 |
+
return {
|
| 640 |
+
"complexity": "no_agents_available",
|
| 641 |
+
"plan": "no_agents_available",
|
| 642 |
+
"plan_instructions": {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
|
| 643 |
+
}
|
| 644 |
+
|
| 645 |
+
try:
|
| 646 |
+
complexity = self.allocator(goal=goal, planner_desc=str(self.planner_desc), dataset=str(dataset))
|
| 647 |
+
logger.log_message(f"Complexity determined: {complexity.exact_word_complexity.strip()}", level=logging.INFO)
|
| 648 |
+
|
| 649 |
+
# If complexity is unrelated, return basic_qa_agent
|
| 650 |
+
if complexity.exact_word_complexity.strip() == "unrelated":
|
| 651 |
+
logger.log_message("Query classified as unrelated, using basic_qa_agent", level=logging.INFO)
|
| 652 |
+
return {
|
| 653 |
+
"complexity": complexity.exact_word_complexity.strip(),
|
| 654 |
+
"plan": "basic_qa_agent",
|
| 655 |
+
"plan_instructions": "{'basic_qa_agent':'Not a data related query, please ask a data related-query'}"
|
| 656 |
+
}
|
| 657 |
+
|
| 658 |
+
# Try to get plan with determined complexity
|
| 659 |
try:
|
| 660 |
+
logger.log_message(f"Attempting to plan with complexity: {complexity.exact_word_complexity.strip()}", level=logging.DEBUG)
|
| 661 |
plan = await self.planners[complexity.exact_word_complexity.strip()](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
|
| 662 |
+
logger.log_message(f"Plan generated successfully: {plan}", level=logging.DEBUG)
|
| 663 |
+
|
| 664 |
+
# Check if the planner returned no_agents_available
|
| 665 |
+
if hasattr(plan, 'plan') and 'no_agents_available' in str(plan.plan):
|
| 666 |
+
logger.log_message("Planner returned no_agents_available", level=logging.WARNING)
|
| 667 |
+
output = {
|
| 668 |
+
"complexity": "no_agents_available",
|
| 669 |
+
"plan": "no_agents_available",
|
| 670 |
+
"plan_instructions": {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
|
| 671 |
+
}
|
| 672 |
+
else:
|
| 673 |
+
output = {
|
| 674 |
+
"complexity": complexity.exact_word_complexity.strip(),
|
| 675 |
+
"plan": dict(plan)
|
| 676 |
+
}
|
| 677 |
|
| 678 |
except Exception as e:
|
| 679 |
+
logger.log_message(f"Error with {complexity.exact_word_complexity.strip()} planner, falling back to intermediate: {str(e)}", level=logging.WARNING)
|
| 680 |
+
|
| 681 |
+
# Fallback to intermediate planner
|
| 682 |
plan = await self.planners["intermediate"](goal=goal, dataset=dataset, Agent_desc=Agent_desc)
|
| 683 |
+
logger.log_message(f"Fallback plan generated: {plan}", level=logging.DEBUG)
|
| 684 |
+
|
| 685 |
+
# Check if the fallback planner also returned no_agents_available
|
| 686 |
+
if hasattr(plan, 'plan') and 'no_agents_available' in str(plan.plan):
|
| 687 |
+
logger.log_message("Fallback planner also returned no_agents_available", level=logging.WARNING)
|
| 688 |
+
output = {
|
| 689 |
+
"complexity": "no_agents_available",
|
| 690 |
+
"plan": "no_agents_available",
|
| 691 |
+
"plan_instructions": {"message": "No agents are currently enabled for analysis. Please enable at least one agent (preprocessing, statistical analysis, machine learning, or visualization) in your template preferences to proceed with data analysis."}
|
| 692 |
+
}
|
| 693 |
+
else:
|
| 694 |
+
output = {
|
| 695 |
+
"complexity": "intermediate",
|
| 696 |
+
"plan": dict(plan)
|
| 697 |
+
}
|
| 698 |
+
|
| 699 |
+
except Exception as e:
|
| 700 |
+
logger.log_message(f"Error in planner forward: {str(e)}", level=logging.ERROR)
|
| 701 |
+
# Return error response
|
| 702 |
+
return {
|
| 703 |
+
"complexity": "error",
|
| 704 |
+
"plan": "basic_qa_agent",
|
| 705 |
+
"plan_instructions": {"error": f"Planning error: {str(e)}"}
|
| 706 |
+
}
|
| 707 |
+
|
| 708 |
+
logger.log_message(f"Final planner output: {output}", level=logging.INFO)
|
| 709 |
return output
|
| 710 |
|
| 711 |
|
| 712 |
|
| 713 |
|
| 714 |
|
| 715 |
+
class preprocessing_agent(dspy.Signature):
|
| 716 |
"""
|
| 717 |
+
You are a preprocessing agent that can work both individually and in multi-agent data analytics systems.
|
| 718 |
You are given:
|
| 719 |
+
* A dataset (already loaded as `df`).
|
| 720 |
+
* A user-defined analysis goal (e.g., predictive modeling, exploration, cleaning).
|
| 721 |
+
* Optional plan instructions that tell you what variables you are expected to create and what variables you are receiving from previous agents.
|
| 722 |
+
|
| 723 |
### Your Responsibilities:
|
| 724 |
+
* If plan_instructions are provided, follow the provided plan and create only the required variables listed in the 'create' section.
|
| 725 |
+
* If no plan_instructions are provided, perform standard data preprocessing based on the goal.
|
| 726 |
+
* Do not create fake data or introduce variables not explicitly part of the instructions.
|
| 727 |
+
* Do not read data from CSV; the dataset (`df`) is already loaded and ready for processing.
|
| 728 |
+
* Generate Python code using NumPy and Pandas to preprocess the data and produce any intermediate variables as specified.
|
| 729 |
+
|
| 730 |
### Best Practices for Preprocessing:
|
| 731 |
+
1. Create a copy of the original DataFrame: It will always be stored as df, it already exists use it!
|
| 732 |
```python
|
| 733 |
processed_df = df.copy()
|
| 734 |
```
|
| 735 |
+
2. Separate column types:
|
| 736 |
```python
|
| 737 |
numeric_cols = processed_df.select_dtypes(include='number').columns
|
| 738 |
categorical_cols = processed_df.select_dtypes(include='object').columns
|
| 739 |
```
|
| 740 |
+
3. Handle missing values:
|
| 741 |
```python
|
| 742 |
for col in numeric_cols:
|
| 743 |
processed_df[col] = processed_df[col].fillna(processed_df[col].median())
|
|
|
|
| 745 |
for col in categorical_cols:
|
| 746 |
processed_df[col] = processed_df[col].fillna(processed_df[col].mode()[0] if not processed_df[col].mode().empty else 'Unknown')
|
| 747 |
```
|
| 748 |
+
4. Convert string columns to datetime safely:
|
| 749 |
```python
|
| 750 |
def safe_to_datetime(x):
|
| 751 |
try:
|
|
|
|
| 755 |
|
| 756 |
cleaned_df['date_column'] = cleaned_df['date_column'].apply(safe_to_datetime)
|
| 757 |
```
|
| 758 |
+
5. Do not alter the DataFrame index unless explicitly instructed.
|
| 759 |
+
6. Log assumptions and corrections in comments to clarify any choices made during preprocessing.
|
| 760 |
+
7. Do not mutate global state: Avoid in-place modifications unless clearly necessary (e.g., using `.copy()`).
|
| 761 |
+
8. Handle data types properly:
|
|
|
|
|
|
|
| 762 |
* Avoid coercing types blindly (e.g., don't compare timestamps to strings or floats).
|
| 763 |
* Use `pd.to_datetime(..., errors='coerce')` for safe datetime parsing.
|
| 764 |
+
9. Preserve column structure: Only drop or rename columns if explicitly instructed.
|
| 765 |
+
|
| 766 |
### Output:
|
| 767 |
+
1. Code: Python code that performs the requested preprocessing steps.
|
| 768 |
+
2. Summary: A brief explanation of what preprocessing was done (e.g., columns handled, missing value treatment).
|
| 769 |
+
|
| 770 |
### Principles to Follow:
|
| 771 |
+
- Never alter the DataFrame index unless explicitly instructed.
|
| 772 |
+
- Handle missing data explicitly, filling with default values when necessary.
|
| 773 |
+
- Preserve column structure and avoid unnecessary modifications.
|
| 774 |
+
- Ensure data types are appropriate (e.g., dates parsed correctly).
|
| 775 |
+
- Log assumptions in the code.
|
| 776 |
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 777 |
"""
|
| 778 |
dataset = dspy.InputField(desc="The dataset, preloaded as df")
|
| 779 |
goal = dspy.InputField(desc="User-defined goal for the analysis")
|
| 780 |
+
plan_instructions = dspy.InputField(desc="Agent-level instructions about what to create and receive (optional for individual use)", default="")
|
| 781 |
|
| 782 |
code = dspy.OutputField(desc="Generated Python code for preprocessing")
|
| 783 |
summary = dspy.OutputField(desc="Explanation of what was done and why")
|
| 784 |
|
| 785 |
+
class data_viz_agent(dspy.Signature):
|
| 786 |
"""
|
| 787 |
+
You are a data visualization agent that can work both individually and in multi-agent analytics pipelines.
|
| 788 |
+
Your primary responsibility is to generate visualizations based on the user-defined goal.
|
| 789 |
+
|
| 790 |
You are provided with:
|
| 791 |
* **goal**: A user-defined goal outlining the type of visualization the user wants (e.g., "plot sales over time with trendline").
|
| 792 |
+
* **dataset**: The dataset (e.g., `df_cleaned`) which will be passed to you by other agents in the pipeline. Do not assume or create any variables — the data is already present and valid when you receive it.
|
| 793 |
* **styling_index**: Specific styling instructions (e.g., axis formatting, color schemes) for the visualization.
|
| 794 |
+
* **plan_instructions**: Optional dictionary containing:
|
| 795 |
+
* **'create'**: List of visualization components you must generate (e.g., 'scatter_plot', 'bar_chart').
|
| 796 |
+
* **'use'**: List of variables you must use to generate the visualizations.
|
| 797 |
+
* **'instructions'**: Additional instructions related to the creation of the visualizations.
|
| 798 |
+
|
| 799 |
+
### Responsibilities:
|
| 800 |
1. **Strict Use of Provided Variables**:
|
| 801 |
+
* You must never create fake data. Only use the variables and datasets that are explicitly provided.
|
| 802 |
+
* If plan_instructions are provided and any variable listed in plan_instructions['use'] is missing, return an error.
|
| 803 |
+
* If no plan_instructions are provided, work with the available dataset directly.
|
| 804 |
+
|
| 805 |
2. **Visualization Creation**:
|
| 806 |
+
* Based on the goal and optional 'create' section of plan_instructions, generate the required visualization using Plotly.
|
| 807 |
+
* Respect the user-defined goal in determining which type of visualization to create.
|
| 808 |
+
|
| 809 |
3. **Performance Optimization**:
|
| 810 |
+
* If the dataset contains more than 50,000 rows, you must sample the data to 5,000 rows to improve performance:
|
| 811 |
```python
|
| 812 |
if len(df) > 50000:
|
| 813 |
df = df.sample(5000, random_state=42)
|
| 814 |
```
|
| 815 |
+
|
| 816 |
4. **Layout and Styling**:
|
| 817 |
+
* Apply formatting and layout adjustments as defined by the styling_index.
|
| 818 |
+
* Ensure that all axes (x and y) have consistent formats (e.g., using `K`, `M`, or 1,000 format, but not mixing formats).
|
| 819 |
+
|
|
|
|
|
|
|
| 820 |
5. **Trendlines**:
|
| 821 |
+
* Trendlines should only be included if explicitly requested in the goal or plan_instructions.
|
| 822 |
+
|
| 823 |
6. **Displaying the Visualization**:
|
| 824 |
* Use Plotly's `fig.show()` method to display the created chart.
|
| 825 |
+
* Never output raw datasets or the goal itself. Only the visualization code and the chart should be returned.
|
| 826 |
+
|
| 827 |
7. **Error Handling**:
|
| 828 |
+
* If required dataset or variables are missing, return an error message indicating which specific variable is missing.
|
| 829 |
+
* If the goal or create instructions are ambiguous, return an error stating the issue.
|
| 830 |
+
|
| 831 |
8. **No Data Modification**:
|
| 832 |
+
* Never modify the provided dataset or generate new data. If the data needs preprocessing, assume it's already been done by other agents.
|
| 833 |
+
|
| 834 |
+
### Important Notes:
|
| 835 |
+
- Use update_yaxes, update_xaxes, not axis
|
| 836 |
+
- Each visualization must be generated as a separate figure using go.Figure()
|
| 837 |
+
- Do NOT use subplots under any circumstances
|
| 838 |
+
- Each figure must be returned individually using: fig.to_html(full_html=False)
|
| 839 |
+
- Use update_layout with xaxis and yaxis only once per figure
|
| 840 |
+
- Enhance readability with low opacity (0.4-0.7) where appropriate
|
| 841 |
+
- Apply visually distinct colors for different elements or categories
|
| 842 |
+
- Use only one number format consistently: either 'K', 'M', or comma-separated values
|
| 843 |
+
- Only include trendlines in scatter plots if the user explicitly asks for them
|
| 844 |
+
- Always end each visualization with: fig.to_html(full_html=False)
|
| 845 |
+
|
| 846 |
+
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 847 |
"""
|
| 848 |
goal = dspy.InputField(desc="User-defined chart goal (e.g. trendlines, scatter plots)")
|
| 849 |
dataset = dspy.InputField(desc="Details of the dataframe (`df`) and its columns")
|
| 850 |
styling_index = dspy.InputField(desc="Instructions for plot styling and layout formatting")
|
| 851 |
+
plan_instructions = dspy.InputField(desc="Variables to create and receive for visualization purposes (optional for individual use)", default="")
|
| 852 |
|
| 853 |
code = dspy.OutputField(desc="Plotly Python code for the visualization")
|
| 854 |
summary = dspy.OutputField(desc="Plain-language summary of what is being visualized")
|
| 855 |
|
| 856 |
+
class statistical_analytics_agent(dspy.Signature):
|
| 857 |
"""
|
| 858 |
+
You are a statistical analytics agent that can work both individually and in multi-agent data analytics pipelines.
|
|
|
|
| 859 |
You are given:
|
| 860 |
* A dataset (usually a cleaned or transformed version like `df_cleaned`).
|
| 861 |
* A user-defined goal (e.g., regression, seasonal decomposition).
|
| 862 |
+
* Optional plan instructions specifying:
|
| 863 |
+
* Which variables you are expected to CREATE (e.g., `regression_model`).
|
| 864 |
+
* Which variables you will USE (e.g., `df_cleaned`, `target_variable`).
|
| 865 |
+
* A set of instructions outlining additional processing or handling for these variables.
|
| 866 |
+
|
| 867 |
+
### Your Responsibilities:
|
| 868 |
* Use the `statsmodels` library to implement the required statistical analysis.
|
| 869 |
* Ensure that all strings are handled as categorical variables via `C(col)` in model formulas.
|
| 870 |
* Always add a constant using `sm.add_constant()`.
|
| 871 |
+
* Do not modify the DataFrame's index.
|
| 872 |
* Convert `X` and `y` to float before fitting the model.
|
| 873 |
* Handle missing values before modeling.
|
| 874 |
* Avoid any data visualization (that is handled by another agent).
|
| 875 |
* Write output to the console using `print()`.
|
| 876 |
+
|
| 877 |
+
### If the goal is regression:
|
| 878 |
* Use `statsmodels.OLS` with proper handling of categorical variables and adding a constant term.
|
| 879 |
* Handle missing values appropriately.
|
| 880 |
+
|
| 881 |
+
### If the goal is seasonal decomposition:
|
| 882 |
* Use `statsmodels.tsa.seasonal_decompose`.
|
| 883 |
* Ensure the time series and period are correctly provided (i.e., `period` should not be `None`).
|
| 884 |
+
|
| 885 |
+
### Instructions to Follow:
|
| 886 |
+
1. If plan_instructions are provided:
|
| 887 |
+
* CREATE only the variables specified in plan_instructions['CREATE']. Do not create any intermediate or new variables.
|
| 888 |
+
* USE only the variables specified in plan_instructions['USE'] to carry out the task.
|
| 889 |
+
* Follow any additional instructions in plan_instructions['INSTRUCTIONS'].
|
| 890 |
+
* Do not reassign or modify any variables passed via plan_instructions.
|
| 891 |
+
2. If no plan_instructions are provided, perform standard statistical analysis based on the goal and available data.
|
| 892 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 893 |
### Example Code Structure:
|
| 894 |
```python
|
| 895 |
import statsmodels.api as sm
|
|
|
|
| 919 |
except Exception as e:
|
| 920 |
return f"An error occurred: {e}"
|
| 921 |
```
|
| 922 |
+
|
| 923 |
+
### Summary:
|
| 924 |
+
1. Always USE the variables passed in plan_instructions['USE'] to carry out the task (if provided).
|
| 925 |
+
2. Only CREATE the variables specified in plan_instructions['CREATE'] (if provided).
|
| 926 |
+
3. Follow any additional instructions in plan_instructions['INSTRUCTIONS'] (if provided).
|
| 927 |
4. Ensure reproducibility by setting the random state appropriately and handling categorical variables.
|
| 928 |
5. Focus on statistical analysis and avoid any unnecessary data manipulation.
|
| 929 |
+
|
| 930 |
+
### Output:
|
| 931 |
+
* The code implementing the statistical analysis, including all required steps.
|
| 932 |
+
* A summary of what the statistical analysis does, how it's performed, and why it fits the goal.
|
| 933 |
* Respond in the user's language for all summary and reasoning but keep the code in english
|
| 934 |
"""
|
| 935 |
dataset = dspy.InputField(desc="Preprocessed dataset, often named df_cleaned")
|
| 936 |
goal = dspy.InputField(desc="The user's statistical analysis goal, e.g., regression or seasonal_decompose")
|
| 937 |
+
plan_instructions = dspy.InputField(desc="Instructions on variables to create and receive for statistical modeling (optional for individual use)", default="")
|
| 938 |
|
| 939 |
code = dspy.OutputField(desc="Python code for statistical modeling using statsmodels")
|
| 940 |
summary = dspy.OutputField(desc="A concise bullet-point summary of the statistical analysis performed and key findings")
|
| 941 |
|
| 942 |
+
class sk_learn_agent(dspy.Signature):
|
|
|
|
|
|
|
| 943 |
"""
|
| 944 |
+
You are a machine learning agent that can work both individually and in multi-agent data analytics pipelines.
|
|
|
|
| 945 |
You are given:
|
| 946 |
* A dataset (often cleaned and feature-engineered).
|
| 947 |
* A user-defined goal (e.g., classification, regression, clustering).
|
| 948 |
+
* Optional plan instructions specifying:
|
| 949 |
+
* Which variables you are expected to CREATE (e.g., `trained_model`, `predictions`).
|
| 950 |
+
* Which variables you will USE (e.g., `df_cleaned`, `target_variable`, `feature_columns`).
|
| 951 |
+
* A set of instructions outlining additional processing or handling for these variables.
|
| 952 |
+
|
| 953 |
+
### Your Responsibilities:
|
| 954 |
* Use the scikit-learn library to implement the appropriate ML pipeline.
|
| 955 |
* Always split data into training and testing sets where applicable.
|
| 956 |
* Use `print()` for all outputs.
|
| 957 |
* Ensure your code is:
|
| 958 |
+
* Reproducible: Set `random_state=42` wherever applicable.
|
| 959 |
+
* Modular: Avoid deeply nested code.
|
| 960 |
+
* Focused on model building, not visualization (leave plotting to the `data_viz_agent`).
|
| 961 |
* Your task may include:
|
| 962 |
* Preprocessing inputs (e.g., encoding).
|
| 963 |
* Model selection and training.
|
| 964 |
* Evaluation (e.g., accuracy, RMSE, classification report).
|
| 965 |
+
|
| 966 |
+
### You must not:
|
| 967 |
* Visualize anything (that's another agent's job).
|
| 968 |
+
* Rely on hardcoded column names — use those passed via plan_instructions or infer from data.
|
| 969 |
+
* Never create or modify any variables not explicitly mentioned in plan_instructions['CREATE'] (if provided).
|
| 970 |
+
* Never create the `df` variable. You will only work with the variables passed via the plan_instructions.
|
| 971 |
+
* Do not introduce intermediate variables unless they are listed in plan_instructions['CREATE'] (if provided).
|
| 972 |
+
|
| 973 |
+
### Instructions to Follow:
|
| 974 |
+
1. If plan_instructions are provided:
|
| 975 |
+
* CREATE only the variables specified in the plan_instructions['CREATE'] list.
|
| 976 |
+
* USE only the variables specified in the plan_instructions['USE'] list.
|
| 977 |
+
* Follow any processing instructions in the plan_instructions['INSTRUCTIONS'] list.
|
| 978 |
+
* Do not reassign or modify any variables passed via plan_instructions.
|
| 979 |
+
2. If no plan_instructions are provided, perform standard machine learning analysis based on the goal and available data.
|
| 980 |
+
|
| 981 |
+
### Example Workflow:
|
| 982 |
+
Given that the plan_instructions specifies variables to CREATE and USE, and includes instructions, your approach should look like this:
|
| 983 |
+
1. Use `df_cleaned` and `feature_columns` from the plan_instructions to extract your features (`X`).
|
| 984 |
+
2. Use `target_column` from plan_instructions to extract your target (`y`).
|
| 985 |
3. If instructions are provided (e.g., scale or encode), follow them.
|
| 986 |
4. Split data into training and testing sets using `train_test_split`.
|
| 987 |
5. Train the model based on the received goal (classification, regression, etc.).
|
| 988 |
+
6. Store the output variables as specified in plan_instructions['CREATE'].
|
| 989 |
+
|
| 990 |
+
### Summary:
|
| 991 |
+
1. Always USE the variables passed in plan_instructions['USE'] to build the pipeline (if provided).
|
| 992 |
+
2. Only CREATE the variables specified in plan_instructions['CREATE'] (if provided).
|
| 993 |
+
3. Follow any additional instructions in plan_instructions['INSTRUCTIONS'] (if provided).
|
| 994 |
+
4. Ensure reproducibility by setting random_state=42 wherever necessary.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 995 |
5. Focus on model building, evaluation, and saving the required outputs—avoid any unnecessary variables.
|
| 996 |
+
|
| 997 |
+
### Output:
|
| 998 |
+
* The code implementing the ML task, including all required steps.
|
| 999 |
+
* A summary of what the model does, how it is evaluated, and why it fits the goal.
|
| 1000 |
* Respond in the user's language for all summary and reasoning but keep the code in english
|
| 1001 |
"""
|
| 1002 |
dataset = dspy.InputField(desc="Input dataset, often cleaned and feature-selected (e.g., df_cleaned)")
|
| 1003 |
goal = dspy.InputField(desc="The user's machine learning goal (e.g., classification or regression)")
|
| 1004 |
+
plan_instructions = dspy.InputField(desc="Instructions indicating what to create and what variables to receive (optional for individual use)", default="")
|
| 1005 |
|
| 1006 |
code = dspy.OutputField(desc="Scikit-learn based machine learning code")
|
| 1007 |
summary = dspy.OutputField(desc="Explanation of the ML approach and evaluation")
|
|
|
|
| 1015 |
goal = dspy.InputField(desc="The user defined goal ")
|
| 1016 |
refined_goal = dspy.OutputField(desc='Refined goal that helps the planner agent plan better')
|
| 1017 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1018 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1019 |
|
| 1020 |
|
| 1021 |
|
|
|
|
| 1052 |
|
| 1053 |
|
| 1054 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1055 |
class code_fix(dspy.Signature):
|
| 1056 |
"""
|
| 1057 |
You are an expert AI developer and data analyst assistant, skilled at identifying and resolving issues in Python code related to data analytics. Another agent has attempted to generate Python code for a data analytics task but produced code that is broken or throws an error.
|
|
|
|
| 1127 |
self.agent_inputs = {}
|
| 1128 |
self.agent_desc = []
|
| 1129 |
|
| 1130 |
+
# If no agents provided, load core agents from database
|
| 1131 |
+
if not agents:
|
| 1132 |
+
# Load the 4 core agents from database
|
| 1133 |
+
core_agent_names = ['preprocessing_agent', 'statistical_analytics_agent', 'sk_learn_agent', 'data_viz_agent']
|
| 1134 |
+
|
| 1135 |
+
for agent_name in core_agent_names:
|
| 1136 |
+
# Get the agent signature class
|
| 1137 |
+
if agent_name == 'preprocessing_agent':
|
| 1138 |
+
agent_signature = preprocessing_agent
|
| 1139 |
+
elif agent_name == 'statistical_analytics_agent':
|
| 1140 |
+
agent_signature = statistical_analytics_agent
|
| 1141 |
+
elif agent_name == 'sk_learn_agent':
|
| 1142 |
+
agent_signature = sk_learn_agent
|
| 1143 |
+
elif agent_name == 'data_viz_agent':
|
| 1144 |
+
agent_signature = data_viz_agent
|
| 1145 |
+
|
| 1146 |
+
# Add to agents dict
|
| 1147 |
+
self.agents[agent_name] = dspy.asyncify(dspy.ChainOfThought(agent_signature))
|
| 1148 |
+
|
| 1149 |
+
# Set input fields based on signature
|
| 1150 |
+
if agent_name == 'data_viz_agent':
|
| 1151 |
+
self.agent_inputs[agent_name] = {'goal', 'dataset', 'styling_index', 'plan_instructions'}
|
| 1152 |
+
else:
|
| 1153 |
+
self.agent_inputs[agent_name] = {'dataset', 'goal', 'plan_instructions'}
|
| 1154 |
+
|
| 1155 |
+
# Get description from database
|
| 1156 |
+
self.agent_desc.append({agent_name: get_agent_description(agent_name)})
|
| 1157 |
+
else:
|
| 1158 |
+
# Load standard agents from provided list (legacy support)
|
| 1159 |
+
for i, a in enumerate(agents):
|
| 1160 |
+
name = a.__pydantic_core_schema__['schema']['model_name']
|
| 1161 |
+
self.agents[name] = dspy.asyncify(dspy.ChainOfThought(a))
|
| 1162 |
+
self.agent_inputs[name] = {x.strip() for x in str(agents[i].__pydantic_core_schema__['cls']).split('->')[0].split('(')[1].split(',')}
|
| 1163 |
+
logger.log_message(f"Added agent: {name}, inputs: {self.agent_inputs[name]}", level=logging.DEBUG)
|
| 1164 |
+
self.agent_desc.append({name: get_agent_description(name)})
|
| 1165 |
|
| 1166 |
+
# Load user-enabled template agents if user_id and db_session are provided
|
| 1167 |
+
if user_id and db_session:
|
| 1168 |
try:
|
| 1169 |
+
# For individual use, load all available templates (not just planner-enabled ones)
|
| 1170 |
+
template_signatures = load_user_enabled_templates_from_db(user_id, db_session)
|
| 1171 |
|
| 1172 |
for template_name, signature in template_signatures.items():
|
| 1173 |
# Add template agent to agents dict
|
| 1174 |
+
self.agents[template_name] = dspy.asyncify(dspy.ChainOfThought(signature))
|
| 1175 |
|
| 1176 |
+
# Extract input fields from signature - templates use standard fields like data_viz_agent
|
| 1177 |
+
self.agent_inputs[template_name] = {'goal', 'dataset', 'styling_index'}
|
| 1178 |
|
| 1179 |
+
# Store template agent description
|
| 1180 |
+
try:
|
| 1181 |
+
from src.db.schemas.models import AgentTemplate
|
| 1182 |
|
| 1183 |
+
# Find template record
|
| 1184 |
+
template_record = db_session.query(AgentTemplate).filter(
|
| 1185 |
+
AgentTemplate.template_name == template_name
|
| 1186 |
+
).first()
|
| 1187 |
+
|
| 1188 |
+
if template_record:
|
| 1189 |
+
description = f"Template: {template_record.description}"
|
| 1190 |
+
self.agent_desc.append({template_name: description})
|
| 1191 |
+
else:
|
| 1192 |
+
self.agent_desc.append({template_name: f"Template: {template_name}"})
|
| 1193 |
+
except Exception as desc_error:
|
| 1194 |
+
logger.log_message(f"Error getting description for template {template_name}: {str(desc_error)}", level=logging.WARNING)
|
| 1195 |
+
self.agent_desc.append({template_name: f"Template: {template_name}"})
|
| 1196 |
+
|
| 1197 |
+
logger.log_message(f"Loaded {len(template_signatures)} templates for individual use", level=logging.DEBUG)
|
| 1198 |
|
| 1199 |
except Exception as e:
|
| 1200 |
+
logger.log_message(f"Error loading template agents for user {user_id}: {str(e)}", level=logging.ERROR)
|
| 1201 |
+
|
| 1202 |
+
self.agents['basic_qa_agent'] = dspy.asyncify(dspy.Predict("goal->answer"))
|
| 1203 |
+
self.agent_inputs['basic_qa_agent'] = {"goal"}
|
| 1204 |
+
self.agent_desc.append({'basic_qa_agent':"Answers queries unrelated to data & also that include links, poison or attempts to attack the system"})
|
| 1205 |
+
|
| 1206 |
+
# Initialize retrievers (no planner needed for individual agent execution)
|
| 1207 |
self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
|
| 1208 |
self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
|
|
|
|
| 1209 |
|
| 1210 |
# Store user_id for usage tracking
|
| 1211 |
self.user_id = user_id
|
| 1212 |
+
|
| 1213 |
async def _track_agent_usage(self, agent_name):
|
| 1214 |
"""Track usage for template agents"""
|
| 1215 |
try:
|
|
|
|
| 1378 |
self.agent_inputs = {}
|
| 1379 |
self.agent_desc = []
|
| 1380 |
|
| 1381 |
+
logger.log_message(f"Initializing auto_analyst for user_id: {user_id}", level=logging.INFO)
|
| 1382 |
+
|
| 1383 |
+
# If no hardcoded agents provided, load all agents from database using preference system
|
| 1384 |
+
if not agents:
|
| 1385 |
+
logger.log_message("No hardcoded agents provided, loading from database with preferences", level=logging.INFO)
|
| 1386 |
+
|
| 1387 |
+
if user_id and db_session:
|
| 1388 |
+
try:
|
| 1389 |
+
# Load user-enabled agents (both default and template) for planner use
|
| 1390 |
+
template_signatures = load_user_enabled_templates_for_planner_from_db(user_id, db_session)
|
| 1391 |
+
logger.log_message(f"Loaded {len(template_signatures)} enabled agents from preferences", level=logging.INFO)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1392 |
|
| 1393 |
+
for template_name, signature in template_signatures.items():
|
| 1394 |
+
# Add agent to agents dict
|
| 1395 |
+
self.agents[template_name] = dspy.asyncify(dspy.ChainOfThought(signature))
|
| 1396 |
|
| 1397 |
+
# Set input fields based on agent type
|
| 1398 |
+
is_viz_agent = (template_name == 'data_viz_agent' or
|
| 1399 |
+
'viz' in template_name.lower() or
|
| 1400 |
+
'visual' in template_name.lower() or
|
| 1401 |
+
'plot' in template_name.lower() or
|
| 1402 |
+
'chart' in template_name.lower() or
|
| 1403 |
+
'matplotlib' in template_name.lower())
|
| 1404 |
|
| 1405 |
+
if is_viz_agent:
|
| 1406 |
+
self.agent_inputs[template_name] = {'goal', 'dataset', 'styling_index', 'plan_instructions'}
|
|
|
|
| 1407 |
else:
|
| 1408 |
+
self.agent_inputs[template_name] = {'dataset', 'goal', 'plan_instructions'}
|
|
|
|
|
|
|
|
|
|
| 1409 |
|
| 1410 |
+
# Get description from database
|
| 1411 |
+
self.agent_desc.append({template_name: get_agent_description(template_name)})
|
| 1412 |
+
logger.log_message(f"Added agent: {template_name}", level=logging.DEBUG)
|
| 1413 |
+
|
| 1414 |
+
logger.log_message(f"Successfully loaded {len(template_signatures)} agents for planner", level=logging.INFO)
|
| 1415 |
+
|
| 1416 |
+
except Exception as e:
|
| 1417 |
+
logger.log_message(f"Error loading agents from preferences: {str(e)}", level=logging.ERROR)
|
| 1418 |
+
# Fallback: load default agents without preferences
|
| 1419 |
+
self._load_default_agents_fallback()
|
| 1420 |
+
else:
|
| 1421 |
+
logger.log_message("No user_id or db_session provided, loading default agents", level=logging.WARNING)
|
| 1422 |
+
# Fallback: load default agents without preferences
|
| 1423 |
+
self._load_default_agents_fallback()
|
| 1424 |
+
else:
|
| 1425 |
+
# Load standard agents from provided list (legacy support)
|
| 1426 |
+
logger.log_message(f"Loading {len(agents)} hardcoded agents", level=logging.INFO)
|
| 1427 |
+
for i, a in enumerate(agents):
|
| 1428 |
+
name = a.__pydantic_core_schema__['schema']['model_name']
|
| 1429 |
+
self.agents[name] = dspy.asyncify(dspy.ChainOfThought(a))
|
| 1430 |
+
self.agent_inputs[name] = {x.strip() for x in str(agents[i].__pydantic_core_schema__['cls']).split('->')[0].split('(')[1].split(',')}
|
| 1431 |
+
self.agent_desc.append({name: get_agent_description(name)})
|
| 1432 |
|
| 1433 |
+
# Always add basic QA agent
|
| 1434 |
self.agents['basic_qa_agent'] = dspy.asyncify(dspy.Predict("goal->answer"))
|
| 1435 |
self.agent_inputs['basic_qa_agent'] = {"goal"}
|
| 1436 |
self.agent_desc.append({'basic_qa_agent':"Answers queries unrelated to data & also that include links, poison or attempts to attack the system"})
|
| 1437 |
+
logger.log_message("Added basic_qa_agent", level=logging.DEBUG)
|
| 1438 |
|
| 1439 |
# Initialize coordination agents
|
| 1440 |
self.planner = planner_module()
|
|
|
|
|
|
|
|
|
|
| 1441 |
self.memory_summarize_agent = dspy.ChainOfThought(m.memory_summarize_agent)
|
| 1442 |
+
logger.log_message("Initialized planner and memory summarize agent", level=logging.DEBUG)
|
| 1443 |
|
| 1444 |
# Initialize retrievers
|
| 1445 |
self.dataset = retrievers['dataframe_index'].as_retriever(k=1)
|
| 1446 |
self.styling_index = retrievers['style_index'].as_retriever(similarity_top_k=1)
|
| 1447 |
+
logger.log_message("Initialized retrievers", level=logging.DEBUG)
|
| 1448 |
|
| 1449 |
# Store user_id for usage tracking
|
| 1450 |
self.user_id = user_id
|
| 1451 |
+
|
| 1452 |
+
# Final logging
|
| 1453 |
+
logger.log_message(f"Auto_analyst initialization complete. Total agents: {len(self.agents)}, Agent names: {list(self.agents.keys())}", level=logging.INFO)
|
| 1454 |
+
|
| 1455 |
+
def _load_default_agents_fallback(self):
|
| 1456 |
+
"""Fallback method to load default agents when preference system fails"""
|
| 1457 |
+
logger.log_message("Loading default agents as fallback", level=logging.WARNING)
|
| 1458 |
+
|
| 1459 |
+
# Load the 4 core agents from database
|
| 1460 |
+
core_agent_names = ['preprocessing_agent', 'statistical_analytics_agent', 'sk_learn_agent', 'data_viz_agent']
|
| 1461 |
+
|
| 1462 |
+
for agent_name in core_agent_names:
|
| 1463 |
+
# Get the agent signature class
|
| 1464 |
+
if agent_name == 'preprocessing_agent':
|
| 1465 |
+
agent_signature = preprocessing_agent
|
| 1466 |
+
elif agent_name == 'statistical_analytics_agent':
|
| 1467 |
+
agent_signature = statistical_analytics_agent
|
| 1468 |
+
elif agent_name == 'sk_learn_agent':
|
| 1469 |
+
agent_signature = sk_learn_agent
|
| 1470 |
+
elif agent_name == 'data_viz_agent':
|
| 1471 |
+
agent_signature = data_viz_agent
|
| 1472 |
+
|
| 1473 |
+
# Add to agents dict
|
| 1474 |
+
self.agents[agent_name] = dspy.asyncify(dspy.ChainOfThought(agent_signature))
|
| 1475 |
+
|
| 1476 |
+
# Set input fields based on signature
|
| 1477 |
+
if agent_name == 'data_viz_agent':
|
| 1478 |
+
self.agent_inputs[agent_name] = {'goal', 'dataset', 'styling_index', 'plan_instructions'}
|
| 1479 |
+
else:
|
| 1480 |
+
self.agent_inputs[agent_name] = {'dataset', 'goal', 'plan_instructions'}
|
| 1481 |
+
|
| 1482 |
+
# Get description from database
|
| 1483 |
+
self.agent_desc.append({agent_name: get_agent_description(agent_name)})
|
| 1484 |
+
logger.log_message(f"Added fallback agent: {agent_name}", level=logging.DEBUG)
|
| 1485 |
|
| 1486 |
async def _track_agent_usage(self, agent_name):
|
| 1487 |
"""Track usage for template agents"""
|
|
|
|
| 1553 |
|
| 1554 |
async def execute_agent(self, agent_name, inputs):
|
| 1555 |
"""Execute a single agent with given inputs"""
|
| 1556 |
+
logger.log_message(f"Executing single agent: {agent_name}", level=logging.DEBUG)
|
| 1557 |
+
|
| 1558 |
try:
|
| 1559 |
result = await self.agents[agent_name.strip()](**inputs)
|
| 1560 |
|
| 1561 |
# Track usage for custom agents and templates
|
| 1562 |
await self._track_agent_usage(agent_name.strip())
|
| 1563 |
|
| 1564 |
+
logger.log_message(f"Agent {agent_name} execution completed", level=logging.DEBUG)
|
| 1565 |
return agent_name.strip(), dict(result)
|
| 1566 |
except Exception as e:
|
| 1567 |
+
logger.log_message(f"Error in execute_agent for {agent_name}: {str(e)}", level=logging.ERROR)
|
| 1568 |
return agent_name.strip(), {"error": str(e)}
|
| 1569 |
|
| 1570 |
async def get_plan(self, query):
|
| 1571 |
"""Get the analysis plan"""
|
| 1572 |
+
logger.log_message(f"Getting plan for query: {query[:100]}...", level=logging.INFO)
|
| 1573 |
+
|
| 1574 |
dict_ = {}
|
| 1575 |
dict_['dataset'] = self.dataset.retrieve(query)[0].text
|
| 1576 |
dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
|
| 1577 |
dict_['goal'] = query
|
| 1578 |
dict_['Agent_desc'] = str(self.agent_desc)
|
| 1579 |
|
| 1580 |
+
logger.log_message(f"Available agents for planning: {list(self.agents.keys())}", level=logging.INFO)
|
| 1581 |
+
logger.log_message(f"Agent descriptions length: {len(self.agent_desc)}", level=logging.DEBUG)
|
| 1582 |
+
|
| 1583 |
+
try:
|
| 1584 |
+
module_return = await self.planner(goal=dict_['goal'], dataset=dict_['dataset'], Agent_desc=dict_['Agent_desc'])
|
| 1585 |
+
|
| 1586 |
+
plan_dict = dict(module_return['plan'])
|
| 1587 |
+
if 'complexity' in module_return:
|
| 1588 |
+
complexity = module_return['complexity']
|
| 1589 |
+
else:
|
| 1590 |
+
complexity = 'basic'
|
| 1591 |
+
plan_dict['complexity'] = complexity
|
| 1592 |
|
| 1593 |
+
return plan_dict
|
| 1594 |
+
|
| 1595 |
+
except Exception as e:
|
| 1596 |
+
logger.log_message(f"Error in get_plan: {str(e)}", level=logging.ERROR)
|
| 1597 |
+
raise
|
| 1598 |
|
| 1599 |
async def execute_plan(self, query, plan):
|
| 1600 |
"""Execute the plan and yield results as they complete"""
|
| 1601 |
+
logger.log_message(f"Executing plan: {plan}", level=logging.INFO)
|
| 1602 |
+
|
| 1603 |
dict_ = {}
|
| 1604 |
dict_['dataset'] = self.dataset.retrieve(query)[0].text
|
| 1605 |
dict_['styling_index'] = self.styling_index.retrieve(query)[0].text
|
|
|
|
| 1610 |
|
| 1611 |
# Clean and split the plan string into agent names
|
| 1612 |
plan_text = plan.get("plan", "").replace("Plan", "").replace(":", "").strip()
|
| 1613 |
+
logger.log_message(f"Plan text after cleaning: {plan_text}", level=logging.DEBUG)
|
| 1614 |
|
| 1615 |
if "basic_qa_agent" in plan_text:
|
| 1616 |
+
logger.log_message("Executing basic_qa_agent", level=logging.INFO)
|
| 1617 |
inputs = dict(goal=query)
|
| 1618 |
+
agent_name, response = await self.execute_agent('basic_qa_agent', inputs)
|
| 1619 |
yield agent_name, inputs, response
|
| 1620 |
return
|
| 1621 |
|
| 1622 |
plan_list = [agent.strip() for agent in plan_text.split("->") if agent.strip()]
|
| 1623 |
+
logger.log_message(f"Plan list: {plan_list}", level=logging.INFO)
|
| 1624 |
|
| 1625 |
# Parse the attached plan_instructions into a dict
|
| 1626 |
raw_instr = plan.get("plan_instructions", {})
|
| 1627 |
if isinstance(raw_instr, str):
|
| 1628 |
try:
|
| 1629 |
plan_instructions = json.loads(raw_instr)
|
| 1630 |
+
except Exception as e:
|
| 1631 |
+
logger.log_message(f"Error parsing plan_instructions JSON: {str(e)}", level=logging.ERROR)
|
| 1632 |
plan_instructions = {}
|
| 1633 |
elif isinstance(raw_instr, dict):
|
| 1634 |
+
plan_instructions = raw_instr
|
| 1635 |
else:
|
| 1636 |
plan_instructions = {}
|
| 1637 |
|
| 1638 |
+
logger.log_message(f"Parsed plan instructions: {plan_instructions}", level=logging.DEBUG)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1639 |
|
| 1640 |
+
# Check if we have no valid agents to execute
|
| 1641 |
+
if not plan_list or all(agent not in self.agents for agent in plan_list):
|
| 1642 |
+
logger.log_message(f"No valid agents found in plan. Available agents: {list(self.agents.keys())}, Plan agents: {plan_list}", level=logging.ERROR)
|
| 1643 |
+
yield "plan_not_found", None, {"error": "No valid agents found in plan"}
|
| 1644 |
+
return
|
| 1645 |
|
| 1646 |
+
# Execute agents in sequence
|
| 1647 |
+
for agent_name in plan_list:
|
| 1648 |
+
if agent_name not in self.agents:
|
| 1649 |
+
logger.log_message(f"Agent '{agent_name}' not found in available agents: {list(self.agents.keys())}", level=logging.ERROR)
|
| 1650 |
+
yield agent_name, {}, {"error": f"Agent '{agent_name}' not available"}
|
| 1651 |
+
continue
|
| 1652 |
+
|
| 1653 |
+
logger.log_message(f"Executing agent: {agent_name}", level=logging.INFO)
|
| 1654 |
|
| 1655 |
+
try:
|
| 1656 |
+
# Prepare inputs for the agent
|
| 1657 |
+
inputs = {x: dict_[x] for x in self.agent_inputs[agent_name] if x in dict_}
|
|
|
|
| 1658 |
|
| 1659 |
+
# Add plan instructions if available for this agent
|
| 1660 |
+
if agent_name in plan_instructions:
|
| 1661 |
+
inputs['plan_instructions'] = json.dumps(plan_instructions[agent_name])
|
| 1662 |
+
else:
|
| 1663 |
+
inputs['plan_instructions'] = ""
|
| 1664 |
|
| 1665 |
+
logger.log_message(f"Agent inputs for {agent_name}: {list(inputs.keys())}", level=logging.DEBUG)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1666 |
|
| 1667 |
+
# Execute the agent
|
| 1668 |
+
agent_result_name, response = await self.execute_agent(agent_name, inputs)
|
| 1669 |
+
logger.log_message(f"Agent {agent_name} completed successfully", level=logging.INFO)
|
|
|
|
|
|
|
| 1670 |
|
| 1671 |
+
yield agent_result_name, inputs, response
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1672 |
|
| 1673 |
+
except Exception as e:
|
| 1674 |
+
logger.log_message(f"Error executing agent {agent_name}: {str(e)}", level=logging.ERROR)
|
| 1675 |
+
yield agent_name, {}, {"error": f"Error executing {agent_name}: {str(e)}"}
|
| 1676 |
|
src/agents/deep_agents.py
CHANGED
|
@@ -728,6 +728,8 @@ Chart Styling Guidelines:
|
|
| 728 |
|
| 729 |
class deep_analysis_module(dspy.Module):
|
| 730 |
def __init__(self,agents, agents_desc):
|
|
|
|
|
|
|
| 731 |
self.agents = agents
|
| 732 |
# Make all dspy operations async using asyncify
|
| 733 |
self.deep_questions = dspy.asyncify(dspy.Predict(deep_questions))
|
|
@@ -741,6 +743,8 @@ class deep_analysis_module(dspy.Module):
|
|
| 741 |
self.styling_instructions = chart_instructions
|
| 742 |
self.agents_desc = agents_desc
|
| 743 |
self.final_conclusion = dspy.asyncify(dspy.ChainOfThought(final_conclusion))
|
|
|
|
|
|
|
| 744 |
|
| 745 |
async def execute_deep_analysis_streaming(self, goal, dataset_info, session_df=None):
|
| 746 |
"""
|
|
@@ -795,7 +799,8 @@ class deep_analysis_module(dspy.Module):
|
|
| 795 |
|
| 796 |
if not all(key in self.agents for key in keys):
|
| 797 |
raise ValueError(f"Invalid agent key(s) in plan instructions. Available agents: {list(self.agents.keys())}")
|
| 798 |
-
|
|
|
|
| 799 |
except (ValueError, SyntaxError, json.JSONDecodeError) as e:
|
| 800 |
try:
|
| 801 |
deep_plan = await self.deep_plan_fixer(plan_instructions=deep_plan.plan_instructions)
|
|
@@ -803,6 +808,8 @@ class deep_analysis_module(dspy.Module):
|
|
| 803 |
if not isinstance(plan_instructions, dict):
|
| 804 |
plan_instructions = json.loads(deep_plan.fixed_plan)
|
| 805 |
keys = [key for key in plan_instructions.keys()]
|
|
|
|
|
|
|
| 806 |
except (ValueError, SyntaxError, json.JSONDecodeError) as e:
|
| 807 |
logger.log_message(f"Error parsing plan instructions: {e}", logging.ERROR)
|
| 808 |
raise e
|
|
@@ -828,19 +835,18 @@ class deep_analysis_module(dspy.Module):
|
|
| 828 |
dspy.Example(
|
| 829 |
goal=questions.deep_questions,
|
| 830 |
dataset=dataset_info,
|
| 831 |
-
|
| 832 |
-
**({"styling_index": "Sample styling guidelines"} if "data_viz" in key else {})
|
| 833 |
).with_inputs(
|
| 834 |
"goal",
|
| 835 |
-
"dataset",
|
| 836 |
-
|
| 837 |
-
*(["styling_index"] if "data_viz" in key else [])
|
| 838 |
)
|
| 839 |
for key in keys
|
| 840 |
]
|
| 841 |
-
|
| 842 |
tasks = [self.agents[key](**q) for q, key in zip(queries, keys)]
|
| 843 |
-
|
| 844 |
# Await all tasks to complete
|
| 845 |
summaries = []
|
| 846 |
codes = []
|
|
|
|
| 728 |
|
| 729 |
class deep_analysis_module(dspy.Module):
|
| 730 |
def __init__(self,agents, agents_desc):
|
| 731 |
+
logger.log_message(f"Initializing deep_analysis_module with {len(agents)} agents: {list(agents.keys())}", level=logging.INFO)
|
| 732 |
+
|
| 733 |
self.agents = agents
|
| 734 |
# Make all dspy operations async using asyncify
|
| 735 |
self.deep_questions = dspy.asyncify(dspy.Predict(deep_questions))
|
|
|
|
| 743 |
self.styling_instructions = chart_instructions
|
| 744 |
self.agents_desc = agents_desc
|
| 745 |
self.final_conclusion = dspy.asyncify(dspy.ChainOfThought(final_conclusion))
|
| 746 |
+
|
| 747 |
+
logger.log_message(f"Deep analysis module initialized successfully with agents: {list(self.agents.keys())}", level=logging.INFO)
|
| 748 |
|
| 749 |
async def execute_deep_analysis_streaming(self, goal, dataset_info, session_df=None):
|
| 750 |
"""
|
|
|
|
| 799 |
|
| 800 |
if not all(key in self.agents for key in keys):
|
| 801 |
raise ValueError(f"Invalid agent key(s) in plan instructions. Available agents: {list(self.agents.keys())}")
|
| 802 |
+
logger.log_message(f"Plan instructions: {plan_instructions}", logging.INFO)
|
| 803 |
+
logger.log_message(f"Keys: {keys}", logging.INFO)
|
| 804 |
except (ValueError, SyntaxError, json.JSONDecodeError) as e:
|
| 805 |
try:
|
| 806 |
deep_plan = await self.deep_plan_fixer(plan_instructions=deep_plan.plan_instructions)
|
|
|
|
| 808 |
if not isinstance(plan_instructions, dict):
|
| 809 |
plan_instructions = json.loads(deep_plan.fixed_plan)
|
| 810 |
keys = [key for key in plan_instructions.keys()]
|
| 811 |
+
logger.log_message(f"Plan instructions fixed: {plan_instructions}", logging.INFO)
|
| 812 |
+
logger.log_message(f"Keys: {keys}", logging.INFO)
|
| 813 |
except (ValueError, SyntaxError, json.JSONDecodeError) as e:
|
| 814 |
logger.log_message(f"Error parsing plan instructions: {e}", logging.ERROR)
|
| 815 |
raise e
|
|
|
|
| 835 |
dspy.Example(
|
| 836 |
goal=questions.deep_questions,
|
| 837 |
dataset=dataset_info,
|
| 838 |
+
plan_instructions=str(plan_instructions[key]),
|
| 839 |
+
**({"styling_index": "Sample styling guidelines"} if "data_viz" in key or "viz" in key.lower() or "visual" in key.lower() or "plot" in key.lower() or "chart" in key.lower() else {})
|
| 840 |
).with_inputs(
|
| 841 |
"goal",
|
| 842 |
+
"dataset",
|
| 843 |
+
"plan_instructions",
|
| 844 |
+
*(["styling_index"] if "data_viz" in key or "viz" in key.lower() or "visual" in key.lower() or "plot" in key.lower() or "chart" in key.lower() else [])
|
| 845 |
)
|
| 846 |
for key in keys
|
| 847 |
]
|
|
|
|
| 848 |
tasks = [self.agents[key](**q) for q, key in zip(queries, keys)]
|
| 849 |
+
|
| 850 |
# Await all tasks to complete
|
| 851 |
summaries = []
|
| 852 |
codes = []
|
src/db/init_db.py
CHANGED
|
@@ -15,15 +15,17 @@ DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///chat_database.db")
|
|
| 15 |
# Determine database type and set appropriate engine configurations
|
| 16 |
if DATABASE_URL.startswith('postgresql'):
|
| 17 |
# PostgreSQL-specific configuration
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
| 27 |
else:
|
| 28 |
# SQLite configuration
|
| 29 |
engine = create_engine(DATABASE_URL)
|
|
|
|
| 15 |
# Determine database type and set appropriate engine configurations
|
| 16 |
if DATABASE_URL.startswith('postgresql'):
|
| 17 |
# PostgreSQL-specific configuration
|
| 18 |
+
ask = input("Are you sure?")
|
| 19 |
+
if ask.lower() == "yes":
|
| 20 |
+
engine = create_engine(
|
| 21 |
+
DATABASE_URL,
|
| 22 |
+
pool_size=10,
|
| 23 |
+
max_overflow=20,
|
| 24 |
+
pool_pre_ping=True, # Check connection validity before use
|
| 25 |
+
pool_recycle=300 # Recycle connections after 5 minutes
|
| 26 |
+
)
|
| 27 |
+
is_postgresql = True
|
| 28 |
+
logger.log_message("Using PostgreSQL database engine", logging.INFO)
|
| 29 |
else:
|
| 30 |
# SQLite configuration
|
| 31 |
engine = create_engine(DATABASE_URL)
|
src/db/init_default_agents.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Initialize default agents in the database.
|
| 3 |
+
This module should be run during application startup to ensure
|
| 4 |
+
default agents are available in the database.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import logging
|
| 8 |
+
from datetime import datetime, UTC
|
| 9 |
+
from src.utils.logger import Logger
|
| 10 |
+
|
| 11 |
+
# Initialize logger
|
| 12 |
+
logger = Logger("init_default_agents", see_time=True, console_log=False)
|
| 13 |
+
|
| 14 |
+
def load_default_agents_to_db(db_session, force_update=False):
|
| 15 |
+
"""
|
| 16 |
+
Load the default agents into the AgentTemplate table.
|
| 17 |
+
|
| 18 |
+
Args:
|
| 19 |
+
db_session: Database session
|
| 20 |
+
force_update: If True, update existing agents. If False, skip existing ones.
|
| 21 |
+
|
| 22 |
+
Returns:
|
| 23 |
+
Tuple (success: bool, message: str)
|
| 24 |
+
"""
|
| 25 |
+
try:
|
| 26 |
+
from src.db.schemas.models import AgentTemplate
|
| 27 |
+
|
| 28 |
+
# Define default agents with their signatures and metadata
|
| 29 |
+
default_agents = {
|
| 30 |
+
"preprocessing_agent": {
|
| 31 |
+
"display_name": "Data Preprocessing Agent",
|
| 32 |
+
"description": "Cleans and prepares a DataFrame using Pandas and NumPy—handles missing values, detects column types, and converts date strings to datetime.",
|
| 33 |
+
"prompt_template": """You are a AI data-preprocessing agent. Generate clean and efficient Python code using NumPy and Pandas to perform introductory data preprocessing on a pre-loaded DataFrame df, based on the user's analysis goals.
|
| 34 |
+
Preprocessing Requirements:
|
| 35 |
+
1. Identify Column Types
|
| 36 |
+
- Separate columns into numeric and categorical using:
|
| 37 |
+
categorical_columns = df.select_dtypes(include=[object, 'category']).columns.tolist()
|
| 38 |
+
numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
|
| 39 |
+
2. Handle Missing Values
|
| 40 |
+
- Numeric columns: Impute missing values using the mean of each column
|
| 41 |
+
- Categorical columns: Impute missing values using the mode of each column
|
| 42 |
+
3. Convert Date Strings to Datetime
|
| 43 |
+
- For any column suspected to represent dates (in string format), convert it to datetime using:
|
| 44 |
+
def safe_to_datetime(date):
|
| 45 |
+
try:
|
| 46 |
+
return pd.to_datetime(date, errors='coerce', cache=False)
|
| 47 |
+
except (ValueError, TypeError):
|
| 48 |
+
return pd.NaT
|
| 49 |
+
df['datetime_column'] = df['datetime_column'].apply(safe_to_datetime)
|
| 50 |
+
- Replace 'datetime_column' with the actual column names containing date-like strings
|
| 51 |
+
Important Notes:
|
| 52 |
+
- Do NOT create a correlation matrix — correlation analysis is outside the scope of preprocessing
|
| 53 |
+
- Do NOT generate any plots or visualizations
|
| 54 |
+
Output Instructions:
|
| 55 |
+
1. Include the full preprocessing Python code
|
| 56 |
+
2. Provide a brief bullet-point summary of the steps performed. Example:
|
| 57 |
+
• Identified 5 numeric and 4 categorical columns
|
| 58 |
+
• Filled missing numeric values with column means
|
| 59 |
+
• Filled missing categorical values with column modes
|
| 60 |
+
• Converted 1 date column to datetime format
|
| 61 |
+
Respond in the user's language for all summary and reasoning but keep the code in english""",
|
| 62 |
+
"category": "Data Manipulation",
|
| 63 |
+
"icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/pandas/pandas-original.svg"
|
| 64 |
+
},
|
| 65 |
+
"statistical_analytics_agent": {
|
| 66 |
+
"display_name": "Statistical Analytics Agent",
|
| 67 |
+
"description": "Performs statistical analysis (e.g., regression, seasonal decomposition) using statsmodels, with proper handling of categorical data and missing values.",
|
| 68 |
+
"prompt_template": """
|
| 69 |
+
You are a statistical analytics agent. Your task is to take a dataset and a user-defined goal and output Python code that performs the appropriate statistical analysis to achieve that goal. Follow these guidelines:
|
| 70 |
+
IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
|
| 71 |
+
Data Handling:
|
| 72 |
+
Always handle strings as categorical variables in a regression using statsmodels C(string_column).
|
| 73 |
+
Do not change the index of the DataFrame.
|
| 74 |
+
Convert X and y into float when fitting a model.
|
| 75 |
+
Error Handling:
|
| 76 |
+
Always check for missing values and handle them appropriately.
|
| 77 |
+
Ensure that categorical variables are correctly processed.
|
| 78 |
+
Provide clear error messages if the model fitting fails.
|
| 79 |
+
Regression:
|
| 80 |
+
For regression, use statsmodels and ensure that a constant term is added to the predictor using sm.add_constant(X).
|
| 81 |
+
Handle categorical variables using C(column_name) in the model formula.
|
| 82 |
+
Fit the model with model = sm.OLS(y.astype(float), X.astype(float)).fit().
|
| 83 |
+
Seasonal Decomposition:
|
| 84 |
+
Ensure the period is set correctly when performing seasonal decomposition.
|
| 85 |
+
Verify the number of observations works for the decomposition.
|
| 86 |
+
Output:
|
| 87 |
+
Ensure the code is executable and as intended.
|
| 88 |
+
Also choose the correct type of model for the problem
|
| 89 |
+
Avoid adding data visualization code.
|
| 90 |
+
Use code like this to prevent failing:
|
| 91 |
+
import pandas as pd
|
| 92 |
+
import numpy as np
|
| 93 |
+
import statsmodels.api as sm
|
| 94 |
+
def statistical_model(X, y, goal, period=None):
|
| 95 |
+
try:
|
| 96 |
+
# Check for missing values and handle them
|
| 97 |
+
X = X.dropna()
|
| 98 |
+
y = y.loc[X.index].dropna()
|
| 99 |
+
# Ensure X and y are aligned
|
| 100 |
+
X = X.loc[y.index]
|
| 101 |
+
# Convert categorical variables
|
| 102 |
+
for col in X.select_dtypes(include=['object', 'category']).columns:
|
| 103 |
+
X[col] = X[col].astype('category')
|
| 104 |
+
# Add a constant term to the predictor
|
| 105 |
+
X = sm.add_constant(X)
|
| 106 |
+
# Fit the model
|
| 107 |
+
if goal == 'regression':
|
| 108 |
+
# Handle categorical variables in the model formula
|
| 109 |
+
formula = 'y ~ ' + ' + '.join([f'C({col})' if X[col].dtype.name == 'category' else col for col in X.columns])
|
| 110 |
+
model = sm.OLS(y.astype(float), X.astype(float)).fit()
|
| 111 |
+
return model.summary()
|
| 112 |
+
elif goal == 'seasonal_decompose':
|
| 113 |
+
if period is None:
|
| 114 |
+
raise ValueError("Period must be specified for seasonal decomposition")
|
| 115 |
+
decomposition = sm.tsa.seasonal_decompose(y, period=period)
|
| 116 |
+
return decomposition
|
| 117 |
+
else:
|
| 118 |
+
raise ValueError("Unknown goal specified. Please provide a valid goal.")
|
| 119 |
+
except Exception as e:
|
| 120 |
+
return f"An error occurred: {e}"
|
| 121 |
+
# Example usage:
|
| 122 |
+
result = statistical_analysis(X, y, goal='regression')
|
| 123 |
+
print(result)
|
| 124 |
+
If visualizing use plotly
|
| 125 |
+
Provide a concise bullet-point summary of the statistical analysis performed.
|
| 126 |
+
|
| 127 |
+
Example Summary:
|
| 128 |
+
• Applied linear regression with OLS to predict house prices based on 5 features
|
| 129 |
+
• Model achieved R-squared of 0.78
|
| 130 |
+
• Significant predictors include square footage (p<0.001) and number of bathrooms (p<0.01)
|
| 131 |
+
• Detected strong seasonal pattern with 12-month periodicity
|
| 132 |
+
• Forecast shows 15% growth trend over next quarter
|
| 133 |
+
Respond in the user's language for all summary and reasoning but keep the code in english""",
|
| 134 |
+
"category": "Statistical Analysis",
|
| 135 |
+
"icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/statsmodels/statsmodels-original.svg"
|
| 136 |
+
},
|
| 137 |
+
"sk_learn_agent": {
|
| 138 |
+
"display_name": "Machine Learning Agent",
|
| 139 |
+
"description": "Trains and evaluates machine learning models using scikit-learn, including classification, regression, and clustering with feature importance insights.",
|
| 140 |
+
"prompt_template": """You are a machine learning agent.
|
| 141 |
+
Your task is to take a dataset and a user-defined goal, and output Python code that performs the appropriate machine learning analysis to achieve that goal.
|
| 142 |
+
You should use the scikit-learn library.
|
| 143 |
+
IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request.
|
| 144 |
+
Make sure your output is as intended!
|
| 145 |
+
Provide a concise bullet-point summary of the machine learning operations performed.
|
| 146 |
+
|
| 147 |
+
Example Summary:
|
| 148 |
+
• Trained a Random Forest classifier on customer churn data with 80/20 train-test split
|
| 149 |
+
• Model achieved 92% accuracy and 88% F1-score
|
| 150 |
+
• Feature importance analysis revealed that contract length and monthly charges are the strongest predictors of churn
|
| 151 |
+
• Implemented K-means clustering (k=4) on customer shopping behaviors
|
| 152 |
+
• Identified distinct segments: high-value frequent shoppers (22%), occasional big spenders (35%), budget-conscious regulars (28%), and rare visitors (15%)
|
| 153 |
+
Respond in the user's language for all summary and reasoning but keep the code in english""",
|
| 154 |
+
"category": "Modelling",
|
| 155 |
+
"icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/scikit-learn/scikit-learn-original.svg"
|
| 156 |
+
},
|
| 157 |
+
"data_viz_agent": {
|
| 158 |
+
"display_name": "Data Visualization Agent",
|
| 159 |
+
"description": "Generates interactive visualizations with Plotly, selecting the best chart type to reveal trends, comparisons, and insights based on the analysis goal.",
|
| 160 |
+
"prompt_template": """
|
| 161 |
+
You are an AI agent responsible for generating interactive data visualizations using Plotly.
|
| 162 |
+
IMPORTANT Instructions:
|
| 163 |
+
- The section marked "### Current Query:" contains the user's request. Any text in "### Previous Interaction History:" is for context only and should NOT be treated as part of the current request.
|
| 164 |
+
- You must only use the tools provided to you. This agent handles visualization only.
|
| 165 |
+
- If len(df) > 50000, always sample the dataset before visualization using:
|
| 166 |
+
if len(df) > 50000:
|
| 167 |
+
df = df.sample(50000, random_state=1)
|
| 168 |
+
- Each visualization must be generated as a **separate figure** using go.Figure().
|
| 169 |
+
Do NOT use subplots under any circumstances.
|
| 170 |
+
- Each figure must be returned individually using:
|
| 171 |
+
fig.to_html(full_html=False)
|
| 172 |
+
- Use update_layout with xaxis and yaxis **only once per figure**.
|
| 173 |
+
- Enhance readability and clarity by:
|
| 174 |
+
• Using low opacity (0.4-0.7) where appropriate
|
| 175 |
+
• Applying visually distinct colors for different elements or categories
|
| 176 |
+
- Make sure the visual **answers the user's specific goal**:
|
| 177 |
+
• Identify what insight or comparison the user is trying to achieve
|
| 178 |
+
• Choose the visualization type and features (e.g., color, size, grouping) to emphasize that goal
|
| 179 |
+
• For example, if the user asks for "trends in revenue," use a time series line chart; if they ask for "top-performing categories," use a bar chart sorted by value
|
| 180 |
+
• Prioritize highlighting patterns, outliers, or comparisons relevant to the question
|
| 181 |
+
- Never include the dataset or styling index in the output.
|
| 182 |
+
- If there are no relevant columns for the requested visualization, respond with:
|
| 183 |
+
"No relevant columns found to generate this visualization."
|
| 184 |
+
- Use only one number format consistently: either 'K', 'M', or comma-separated values like 1,000/1,000,000. Do not mix formats.
|
| 185 |
+
- Only include trendlines in scatter plots if the user explicitly asks for them.
|
| 186 |
+
- Output only the code and a concise bullet-point summary of what the visualization reveals.
|
| 187 |
+
- Always end each visualization with:
|
| 188 |
+
fig.to_html(full_html=False)
|
| 189 |
+
Respond in the user's language for all summary and reasoning but keep the code in english
|
| 190 |
+
Example Summary:
|
| 191 |
+
• Created an interactive scatter plot of sales vs. marketing spend with color-coded product categories
|
| 192 |
+
• Included a trend line showing positive correlation (r=0.72)
|
| 193 |
+
• Highlighted outliers where high marketing spend resulted in low sales
|
| 194 |
+
• Generated a time series chart of monthly revenue from 2020-2023
|
| 195 |
+
• Added annotations for key business events
|
| 196 |
+
• Visualization reveals 35% YoY growth with seasonal peaks in Q4""",
|
| 197 |
+
"category": "Visualization",
|
| 198 |
+
"icon_url": "https://cdn.jsdelivr.net/gh/devicons/devicon/icons/plotly/plotly-original.svg"
|
| 199 |
+
}
|
| 200 |
+
}
|
| 201 |
+
|
| 202 |
+
created_count = 0
|
| 203 |
+
updated_count = 0
|
| 204 |
+
|
| 205 |
+
for template_name, agent_data in default_agents.items():
|
| 206 |
+
# Check if agent already exists
|
| 207 |
+
existing_agent = db_session.query(AgentTemplate).filter(
|
| 208 |
+
AgentTemplate.template_name == template_name
|
| 209 |
+
).first()
|
| 210 |
+
|
| 211 |
+
if existing_agent:
|
| 212 |
+
if force_update:
|
| 213 |
+
# Update existing agent
|
| 214 |
+
existing_agent.display_name = agent_data["display_name"]
|
| 215 |
+
existing_agent.description = agent_data["description"]
|
| 216 |
+
existing_agent.prompt_template = agent_data["prompt_template"]
|
| 217 |
+
existing_agent.category = agent_data["category"]
|
| 218 |
+
existing_agent.icon_url = agent_data["icon_url"]
|
| 219 |
+
existing_agent.is_premium_only = False
|
| 220 |
+
existing_agent.is_active = True
|
| 221 |
+
existing_agent.updated_at = datetime.now(UTC)
|
| 222 |
+
updated_count += 1
|
| 223 |
+
else:
|
| 224 |
+
logger.log_message(f"Agent '{template_name}' already exists, skipping", level=logging.INFO)
|
| 225 |
+
continue
|
| 226 |
+
else:
|
| 227 |
+
# Create new agent
|
| 228 |
+
new_agent = AgentTemplate(
|
| 229 |
+
template_name=template_name,
|
| 230 |
+
display_name=agent_data["display_name"],
|
| 231 |
+
description=agent_data["description"],
|
| 232 |
+
prompt_template=agent_data["prompt_template"],
|
| 233 |
+
category=agent_data["category"],
|
| 234 |
+
icon_url=agent_data["icon_url"],
|
| 235 |
+
is_premium_only=False,
|
| 236 |
+
is_active=True,
|
| 237 |
+
created_at=datetime.now(UTC),
|
| 238 |
+
updated_at=datetime.now(UTC)
|
| 239 |
+
)
|
| 240 |
+
db_session.add(new_agent)
|
| 241 |
+
created_count += 1
|
| 242 |
+
|
| 243 |
+
db_session.commit()
|
| 244 |
+
|
| 245 |
+
message = f"Successfully loaded default agents. Created: {created_count}, Updated: {updated_count}"
|
| 246 |
+
logger.log_message(message, level=logging.INFO)
|
| 247 |
+
return True, message
|
| 248 |
+
|
| 249 |
+
except Exception as e:
|
| 250 |
+
db_session.rollback()
|
| 251 |
+
error_msg = f"Error loading default agents: {str(e)}"
|
| 252 |
+
logger.log_message(error_msg, level=logging.ERROR)
|
| 253 |
+
return False, error_msg
|
| 254 |
+
|
| 255 |
+
def initialize_default_agents(force_update=False):
|
| 256 |
+
"""
|
| 257 |
+
Initialize default agents during application startup.
|
| 258 |
+
|
| 259 |
+
Args:
|
| 260 |
+
force_update: If True, update existing agents. If False, skip existing ones.
|
| 261 |
+
|
| 262 |
+
Returns:
|
| 263 |
+
bool: True if successful, False otherwise
|
| 264 |
+
"""
|
| 265 |
+
try:
|
| 266 |
+
from src.db.init_db import session_factory
|
| 267 |
+
|
| 268 |
+
session = session_factory()
|
| 269 |
+
try:
|
| 270 |
+
success, message = load_default_agents_to_db(session, force_update=force_update)
|
| 271 |
+
logger.log_message(f"Default agents initialization: {message}", level=logging.INFO)
|
| 272 |
+
return success
|
| 273 |
+
finally:
|
| 274 |
+
session.close()
|
| 275 |
+
|
| 276 |
+
except Exception as e:
|
| 277 |
+
logger.log_message(f"Failed to initialize default agents: {str(e)}", level=logging.ERROR)
|
| 278 |
+
return False
|
| 279 |
+
|
| 280 |
+
if __name__ == "__main__":
|
| 281 |
+
initialize_default_agents(force_update=True)
|
src/managers/session_manager.py
CHANGED
|
@@ -27,19 +27,21 @@ class SessionManager:
|
|
| 27 |
|
| 28 |
def __init__(self, styling_instructions: List[str], available_agents: Dict):
|
| 29 |
"""
|
| 30 |
-
Initialize
|
| 31 |
|
| 32 |
Args:
|
| 33 |
-
styling_instructions: List of styling instructions
|
| 34 |
-
available_agents: Dictionary of available agents
|
| 35 |
"""
|
|
|
|
| 36 |
self._sessions = {}
|
| 37 |
self._default_df = None
|
| 38 |
self._default_retrievers = None
|
| 39 |
self._default_ai_system = None
|
| 40 |
-
self._dataset_description = None
|
| 41 |
self._make_data = None
|
| 42 |
-
|
|
|
|
|
|
|
| 43 |
|
| 44 |
self._dataset_description = """This dataset contains residential property information with details about pricing, physical characteristics, and amenities. The data can be used for real estate market analysis, property valuation, and understanding the relationship between house features and prices.
|
| 45 |
|
|
@@ -92,8 +94,8 @@ This dataset appears clean with consistent formatting and no missing values, mak
|
|
| 92 |
self._default_df = pd.read_csv("Housing.csv")
|
| 93 |
self._make_data = make_data(self._default_df, self._dataset_description)
|
| 94 |
self._default_retrievers = self.initialize_retrievers(self.styling_instructions, [str(self._make_data)])
|
| 95 |
-
|
| 96 |
-
|
| 97 |
except Exception as e:
|
| 98 |
logger.log_message(f"Error initializing default dataset: {str(e)}", level=logging.ERROR)
|
| 99 |
raise e
|
|
@@ -311,7 +313,7 @@ This dataset appears clean with consistent formatting and no missing values, mak
|
|
| 311 |
try:
|
| 312 |
# Create AI system with user context to load custom agents
|
| 313 |
ai_system = auto_analyst(
|
| 314 |
-
agents=
|
| 315 |
retrievers=retrievers,
|
| 316 |
user_id=user_id,
|
| 317 |
db_session=db_session
|
|
@@ -322,12 +324,12 @@ This dataset appears clean with consistent formatting and no missing values, mak
|
|
| 322 |
db_session.close()
|
| 323 |
else:
|
| 324 |
# Create standard AI system without custom agents
|
| 325 |
-
return auto_analyst(agents=
|
| 326 |
|
| 327 |
except Exception as e:
|
| 328 |
logger.log_message(f"Error creating AI system for user {user_id}: {str(e)}", level=logging.ERROR)
|
| 329 |
# Fallback to standard AI system
|
| 330 |
-
return auto_analyst(agents=
|
| 331 |
|
| 332 |
def set_session_user(self, session_id: str, user_id: int, chat_id: int = None):
|
| 333 |
"""
|
|
|
|
| 27 |
|
| 28 |
def __init__(self, styling_instructions: List[str], available_agents: Dict):
|
| 29 |
"""
|
| 30 |
+
Initialize SessionManager with styling instructions and available agents
|
| 31 |
|
| 32 |
Args:
|
| 33 |
+
styling_instructions: List of styling instructions for visualization
|
| 34 |
+
available_agents: Dictionary of available agents (deprecated - agents now loaded from DB)
|
| 35 |
"""
|
| 36 |
+
self.styling_instructions = styling_instructions
|
| 37 |
self._sessions = {}
|
| 38 |
self._default_df = None
|
| 39 |
self._default_retrievers = None
|
| 40 |
self._default_ai_system = None
|
|
|
|
| 41 |
self._make_data = None
|
| 42 |
+
# Initialize chat manager
|
| 43 |
+
self._dataset_description = "Housing Dataset"
|
| 44 |
+
self._default_name = "Housing.csv"
|
| 45 |
|
| 46 |
self._dataset_description = """This dataset contains residential property information with details about pricing, physical characteristics, and amenities. The data can be used for real estate market analysis, property valuation, and understanding the relationship between house features and prices.
|
| 47 |
|
|
|
|
| 94 |
self._default_df = pd.read_csv("Housing.csv")
|
| 95 |
self._make_data = make_data(self._default_df, self._dataset_description)
|
| 96 |
self._default_retrievers = self.initialize_retrievers(self.styling_instructions, [str(self._make_data)])
|
| 97 |
+
# Create default AI system - agents will be loaded from database
|
| 98 |
+
self._default_ai_system = auto_analyst(agents=[], retrievers=self._default_retrievers)
|
| 99 |
except Exception as e:
|
| 100 |
logger.log_message(f"Error initializing default dataset: {str(e)}", level=logging.ERROR)
|
| 101 |
raise e
|
|
|
|
| 313 |
try:
|
| 314 |
# Create AI system with user context to load custom agents
|
| 315 |
ai_system = auto_analyst(
|
| 316 |
+
agents=[],
|
| 317 |
retrievers=retrievers,
|
| 318 |
user_id=user_id,
|
| 319 |
db_session=db_session
|
|
|
|
| 324 |
db_session.close()
|
| 325 |
else:
|
| 326 |
# Create standard AI system without custom agents
|
| 327 |
+
return auto_analyst(agents=[], retrievers=retrievers)
|
| 328 |
|
| 329 |
except Exception as e:
|
| 330 |
logger.log_message(f"Error creating AI system for user {user_id}: {str(e)}", level=logging.ERROR)
|
| 331 |
# Fallback to standard AI system
|
| 332 |
+
return auto_analyst(agents=[], retrievers=retrievers)
|
| 333 |
|
| 334 |
def set_session_user(self, session_id: str, user_id: int, chat_id: int = None):
|
| 335 |
"""
|
src/managers/user_manager.py
CHANGED
|
@@ -1,12 +1,13 @@
|
|
| 1 |
import logging
|
| 2 |
import os
|
| 3 |
from typing import Optional
|
|
|
|
| 4 |
|
| 5 |
from fastapi import Depends, HTTPException, Request, status
|
| 6 |
from fastapi.security import APIKeyHeader
|
| 7 |
|
| 8 |
from src.db.init_db import get_session
|
| 9 |
-
from src.db.schemas.models import User as DBUser
|
| 10 |
from src.schemas.user_schemas import User
|
| 11 |
from src.utils.logger import Logger
|
| 12 |
|
|
@@ -100,6 +101,9 @@ def create_user(username: str, email: str) -> User:
|
|
| 100 |
session.commit()
|
| 101 |
session.refresh(new_user)
|
| 102 |
|
|
|
|
|
|
|
|
|
|
| 103 |
return User(
|
| 104 |
user_id=new_user.user_id,
|
| 105 |
username=new_user.username,
|
|
@@ -131,3 +135,48 @@ def get_user_by_email(email: str) -> Optional[User]:
|
|
| 131 |
return None
|
| 132 |
finally:
|
| 133 |
session.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import logging
|
| 2 |
import os
|
| 3 |
from typing import Optional
|
| 4 |
+
from datetime import datetime, UTC
|
| 5 |
|
| 6 |
from fastapi import Depends, HTTPException, Request, status
|
| 7 |
from fastapi.security import APIKeyHeader
|
| 8 |
|
| 9 |
from src.db.init_db import get_session
|
| 10 |
+
from src.db.schemas.models import User as DBUser, AgentTemplate, UserTemplatePreference
|
| 11 |
from src.schemas.user_schemas import User
|
| 12 |
from src.utils.logger import Logger
|
| 13 |
|
|
|
|
| 101 |
session.commit()
|
| 102 |
session.refresh(new_user)
|
| 103 |
|
| 104 |
+
# Enable default agents for the new user
|
| 105 |
+
_enable_default_agents_for_user(new_user.user_id, session)
|
| 106 |
+
|
| 107 |
return User(
|
| 108 |
user_id=new_user.user_id,
|
| 109 |
username=new_user.username,
|
|
|
|
| 135 |
return None
|
| 136 |
finally:
|
| 137 |
session.close()
|
| 138 |
+
|
| 139 |
+
def _enable_default_agents_for_user(user_id: int, session):
|
| 140 |
+
"""Enable default agents for a new user"""
|
| 141 |
+
try:
|
| 142 |
+
# Get all default agents (the 4 built-in agents)
|
| 143 |
+
default_agent_names = [
|
| 144 |
+
"preprocessing_agent",
|
| 145 |
+
"statistical_analytics_agent",
|
| 146 |
+
"sk_learn_agent",
|
| 147 |
+
"data_viz_agent"
|
| 148 |
+
]
|
| 149 |
+
|
| 150 |
+
# Find these agents in the database
|
| 151 |
+
default_agents = session.query(AgentTemplate).filter(
|
| 152 |
+
AgentTemplate.template_name.in_(default_agent_names),
|
| 153 |
+
AgentTemplate.is_active == True
|
| 154 |
+
).all()
|
| 155 |
+
|
| 156 |
+
# Enable each default agent for the user
|
| 157 |
+
for agent in default_agents:
|
| 158 |
+
# Check if preference already exists
|
| 159 |
+
existing_pref = session.query(UserTemplatePreference).filter(
|
| 160 |
+
UserTemplatePreference.user_id == user_id,
|
| 161 |
+
UserTemplatePreference.template_id == agent.template_id
|
| 162 |
+
).first()
|
| 163 |
+
|
| 164 |
+
if not existing_pref:
|
| 165 |
+
# Create new preference with enabled=True
|
| 166 |
+
new_pref = UserTemplatePreference(
|
| 167 |
+
user_id=user_id,
|
| 168 |
+
template_id=agent.template_id,
|
| 169 |
+
is_enabled=True, # Enable by default
|
| 170 |
+
usage_count=0,
|
| 171 |
+
created_at=datetime.now(UTC),
|
| 172 |
+
updated_at=datetime.now(UTC)
|
| 173 |
+
)
|
| 174 |
+
session.add(new_pref)
|
| 175 |
+
|
| 176 |
+
session.commit()
|
| 177 |
+
logger.log_message(f"Enabled {len(default_agents)} default agents for user {user_id}", level=logging.INFO)
|
| 178 |
+
|
| 179 |
+
except Exception as e:
|
| 180 |
+
session.rollback()
|
| 181 |
+
logger.log_message(f"Error enabling default agents for user {user_id}: {str(e)}", level=logging.ERROR)
|
| 182 |
+
raise
|
src/routes/templates_routes.py
CHANGED
|
@@ -41,12 +41,15 @@ class UserTemplatePreferenceResponse(BaseModel):
|
|
| 41 |
template_category: Optional[str]
|
| 42 |
icon_url: Optional[str]
|
| 43 |
is_premium_only: bool
|
|
|
|
| 44 |
is_enabled: bool
|
| 45 |
usage_count: int
|
| 46 |
last_used_at: Optional[datetime]
|
|
|
|
|
|
|
| 47 |
|
| 48 |
-
class
|
| 49 |
-
is_enabled: bool
|
| 50 |
|
| 51 |
def get_global_usage_counts(session, template_ids: List[int] = None) -> Dict[int, int]:
|
| 52 |
"""
|
|
@@ -140,6 +143,14 @@ async def get_user_template_preferences(user_id: int):
|
|
| 140 |
AgentTemplate.is_active == True
|
| 141 |
).all()
|
| 142 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
result = []
|
| 144 |
for template in templates:
|
| 145 |
# Get user preference for this template if it exists
|
|
@@ -148,6 +159,10 @@ async def get_user_template_preferences(user_id: int):
|
|
| 148 |
UserTemplatePreference.template_id == template.template_id
|
| 149 |
).first()
|
| 150 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
result.append(UserTemplatePreferenceResponse(
|
| 152 |
template_id=template.template_id,
|
| 153 |
template_name=template.template_name,
|
|
@@ -156,9 +171,12 @@ async def get_user_template_preferences(user_id: int):
|
|
| 156 |
template_category=template.category,
|
| 157 |
icon_url=template.icon_url,
|
| 158 |
is_premium_only=template.is_premium_only,
|
| 159 |
-
|
|
|
|
| 160 |
usage_count=preference.usage_count if preference else 0,
|
| 161 |
-
last_used_at=preference.last_used_at if preference else None
|
|
|
|
|
|
|
| 162 |
))
|
| 163 |
|
| 164 |
return result
|
|
@@ -189,6 +207,14 @@ async def get_user_enabled_templates(user_id: int):
|
|
| 189 |
AgentTemplate.is_active == True
|
| 190 |
).all()
|
| 191 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
result = []
|
| 193 |
for template in all_templates:
|
| 194 |
# Check if user has a preference record for this template
|
|
@@ -197,8 +223,12 @@ async def get_user_enabled_templates(user_id: int):
|
|
| 197 |
UserTemplatePreference.template_id == template.template_id
|
| 198 |
).first()
|
| 199 |
|
| 200 |
-
#
|
| 201 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
|
| 203 |
if is_enabled:
|
| 204 |
result.append(UserTemplatePreferenceResponse(
|
|
@@ -209,9 +239,12 @@ async def get_user_enabled_templates(user_id: int):
|
|
| 209 |
template_category=template.category,
|
| 210 |
icon_url=template.icon_url,
|
| 211 |
is_premium_only=template.is_premium_only,
|
|
|
|
| 212 |
is_enabled=True,
|
| 213 |
usage_count=preference.usage_count if preference else 0,
|
| 214 |
-
last_used_at=preference.last_used_at if preference else None
|
|
|
|
|
|
|
| 215 |
))
|
| 216 |
|
| 217 |
return result
|
|
@@ -237,36 +270,66 @@ async def get_user_enabled_templates_for_planner(user_id: int):
|
|
| 237 |
if not user:
|
| 238 |
raise HTTPException(status_code=404, detail="User not found")
|
| 239 |
|
| 240 |
-
# Get
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
).limit(10).all()
|
| 248 |
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
).first()
|
| 256 |
|
| 257 |
-
if template
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 270 |
|
| 271 |
logger.log_message(f"Retrieved {len(result)} enabled templates for planner for user {user_id}", level=logging.INFO)
|
| 272 |
return result
|
|
@@ -281,7 +344,7 @@ async def get_user_enabled_templates_for_planner(user_id: int):
|
|
| 281 |
raise HTTPException(status_code=500, detail=f"Failed to retrieve planner templates: {str(e)}")
|
| 282 |
|
| 283 |
@router.post("/user/{user_id}/template/{template_id}/toggle")
|
| 284 |
-
async def toggle_template_preference(user_id: int, template_id: int, request:
|
| 285 |
"""Toggle a user's template preference (enable/disable for planner use)"""
|
| 286 |
try:
|
| 287 |
session = session_factory()
|
|
|
|
| 41 |
template_category: Optional[str]
|
| 42 |
icon_url: Optional[str]
|
| 43 |
is_premium_only: bool
|
| 44 |
+
is_active: bool
|
| 45 |
is_enabled: bool
|
| 46 |
usage_count: int
|
| 47 |
last_used_at: Optional[datetime]
|
| 48 |
+
created_at: Optional[datetime]
|
| 49 |
+
updated_at: Optional[datetime]
|
| 50 |
|
| 51 |
+
class TogglePreferenceRequest(BaseModel):
|
| 52 |
+
is_enabled: bool
|
| 53 |
|
| 54 |
def get_global_usage_counts(session, template_ids: List[int] = None) -> Dict[int, int]:
|
| 55 |
"""
|
|
|
|
| 143 |
AgentTemplate.is_active == True
|
| 144 |
).all()
|
| 145 |
|
| 146 |
+
# Get list of default agent names that should be enabled by default
|
| 147 |
+
default_agent_names = [
|
| 148 |
+
"preprocessing_agent",
|
| 149 |
+
"statistical_analytics_agent",
|
| 150 |
+
"sk_learn_agent",
|
| 151 |
+
"data_viz_agent"
|
| 152 |
+
]
|
| 153 |
+
|
| 154 |
result = []
|
| 155 |
for template in templates:
|
| 156 |
# Get user preference for this template if it exists
|
|
|
|
| 159 |
UserTemplatePreference.template_id == template.template_id
|
| 160 |
).first()
|
| 161 |
|
| 162 |
+
# Determine if template should be enabled by default
|
| 163 |
+
is_default_agent = template.template_name in default_agent_names
|
| 164 |
+
default_enabled = is_default_agent # Default agents enabled by default, others disabled
|
| 165 |
+
|
| 166 |
result.append(UserTemplatePreferenceResponse(
|
| 167 |
template_id=template.template_id,
|
| 168 |
template_name=template.template_name,
|
|
|
|
| 171 |
template_category=template.category,
|
| 172 |
icon_url=template.icon_url,
|
| 173 |
is_premium_only=template.is_premium_only,
|
| 174 |
+
is_active=template.is_active,
|
| 175 |
+
is_enabled=preference.is_enabled if preference else default_enabled, # Default agents enabled by default
|
| 176 |
usage_count=preference.usage_count if preference else 0,
|
| 177 |
+
last_used_at=preference.last_used_at if preference else None,
|
| 178 |
+
created_at=preference.created_at if preference else None,
|
| 179 |
+
updated_at=preference.updated_at if preference else None
|
| 180 |
))
|
| 181 |
|
| 182 |
return result
|
|
|
|
| 207 |
AgentTemplate.is_active == True
|
| 208 |
).all()
|
| 209 |
|
| 210 |
+
# Get list of default agent names that should be enabled by default
|
| 211 |
+
default_agent_names = [
|
| 212 |
+
"preprocessing_agent",
|
| 213 |
+
"statistical_analytics_agent",
|
| 214 |
+
"sk_learn_agent",
|
| 215 |
+
"data_viz_agent"
|
| 216 |
+
]
|
| 217 |
+
|
| 218 |
result = []
|
| 219 |
for template in all_templates:
|
| 220 |
# Check if user has a preference record for this template
|
|
|
|
| 223 |
UserTemplatePreference.template_id == template.template_id
|
| 224 |
).first()
|
| 225 |
|
| 226 |
+
# Determine if template should be enabled by default
|
| 227 |
+
is_default_agent = template.template_name in default_agent_names
|
| 228 |
+
default_enabled = is_default_agent # Default agents enabled by default, others disabled
|
| 229 |
+
|
| 230 |
+
# Template is enabled by default for default agents, disabled for others
|
| 231 |
+
is_enabled = preference.is_enabled if preference else default_enabled
|
| 232 |
|
| 233 |
if is_enabled:
|
| 234 |
result.append(UserTemplatePreferenceResponse(
|
|
|
|
| 239 |
template_category=template.category,
|
| 240 |
icon_url=template.icon_url,
|
| 241 |
is_premium_only=template.is_premium_only,
|
| 242 |
+
is_active=template.is_active,
|
| 243 |
is_enabled=True,
|
| 244 |
usage_count=preference.usage_count if preference else 0,
|
| 245 |
+
last_used_at=preference.last_used_at if preference else None,
|
| 246 |
+
created_at=preference.created_at if preference else None,
|
| 247 |
+
updated_at=preference.updated_at if preference else None
|
| 248 |
))
|
| 249 |
|
| 250 |
return result
|
|
|
|
| 270 |
if not user:
|
| 271 |
raise HTTPException(status_code=404, detail="User not found")
|
| 272 |
|
| 273 |
+
# Get list of default agent names that should be enabled by default
|
| 274 |
+
default_agent_names = [
|
| 275 |
+
"preprocessing_agent",
|
| 276 |
+
"statistical_analytics_agent",
|
| 277 |
+
"sk_learn_agent",
|
| 278 |
+
"data_viz_agent"
|
| 279 |
+
]
|
|
|
|
| 280 |
|
| 281 |
+
# Get all active templates
|
| 282 |
+
all_templates = session.query(AgentTemplate).filter(
|
| 283 |
+
AgentTemplate.is_active == True
|
| 284 |
+
).all()
|
| 285 |
+
|
| 286 |
+
enabled_templates = []
|
| 287 |
+
for template in all_templates:
|
| 288 |
+
# Check if user has a preference record for this template
|
| 289 |
+
preference = session.query(UserTemplatePreference).filter(
|
| 290 |
+
UserTemplatePreference.user_id == user_id,
|
| 291 |
+
UserTemplatePreference.template_id == template.template_id
|
| 292 |
).first()
|
| 293 |
|
| 294 |
+
# Determine if template should be enabled by default
|
| 295 |
+
is_default_agent = template.template_name in default_agent_names
|
| 296 |
+
default_enabled = is_default_agent # Default agents enabled by default, others disabled
|
| 297 |
+
|
| 298 |
+
# Template is enabled by default for default agents, disabled for others
|
| 299 |
+
is_enabled = preference.is_enabled if preference else default_enabled
|
| 300 |
+
|
| 301 |
+
if is_enabled:
|
| 302 |
+
enabled_templates.append({
|
| 303 |
+
'template': template,
|
| 304 |
+
'preference': preference,
|
| 305 |
+
'usage_count': preference.usage_count if preference else 0,
|
| 306 |
+
'last_used_at': preference.last_used_at if preference else None
|
| 307 |
+
})
|
| 308 |
+
|
| 309 |
+
# Sort by usage (most used first) and limit to 10
|
| 310 |
+
enabled_templates.sort(key=lambda x: (x['usage_count'], x['last_used_at'] or datetime.min.replace(tzinfo=UTC)), reverse=True)
|
| 311 |
+
enabled_templates = enabled_templates[:10]
|
| 312 |
+
|
| 313 |
+
result = []
|
| 314 |
+
for item in enabled_templates:
|
| 315 |
+
template = item['template']
|
| 316 |
+
preference = item['preference']
|
| 317 |
+
|
| 318 |
+
result.append(UserTemplatePreferenceResponse(
|
| 319 |
+
template_id=template.template_id,
|
| 320 |
+
template_name=template.template_name,
|
| 321 |
+
display_name=template.display_name,
|
| 322 |
+
description=template.description,
|
| 323 |
+
template_category=template.category,
|
| 324 |
+
icon_url=template.icon_url,
|
| 325 |
+
is_premium_only=template.is_premium_only,
|
| 326 |
+
is_active=template.is_active,
|
| 327 |
+
is_enabled=True,
|
| 328 |
+
usage_count=preference.usage_count if preference else 0,
|
| 329 |
+
last_used_at=preference.last_used_at if preference else None,
|
| 330 |
+
created_at=preference.created_at if preference else None,
|
| 331 |
+
updated_at=preference.updated_at if preference else None
|
| 332 |
+
))
|
| 333 |
|
| 334 |
logger.log_message(f"Retrieved {len(result)} enabled templates for planner for user {user_id}", level=logging.INFO)
|
| 335 |
return result
|
|
|
|
| 344 |
raise HTTPException(status_code=500, detail=f"Failed to retrieve planner templates: {str(e)}")
|
| 345 |
|
| 346 |
@router.post("/user/{user_id}/template/{template_id}/toggle")
|
| 347 |
+
async def toggle_template_preference(user_id: int, template_id: int, request: TogglePreferenceRequest):
|
| 348 |
"""Toggle a user's template preference (enable/disable for planner use)"""
|
| 349 |
try:
|
| 350 |
session = session_factory()
|