DataVision CI/CD Bot commited on
Commit
58e2a2c
·
0 Parent(s):

release: clean production build for HuggingFace Space

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .env.example +73 -0
  2. .env.production +79 -0
  3. .gitattributes +18 -0
  4. .github/workflows/ci.yml +91 -0
  5. .gitignore +112 -0
  6. .python-version +1 -0
  7. AUTOML_DOCUMENTATION.md +348 -0
  8. Dockerfile +79 -0
  9. HUGGINGFACE_DEPLOYMENT.md +123 -0
  10. MLOPS.md +78 -0
  11. PRD.md +838 -0
  12. README.md +85 -0
  13. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/Dockerfile +12 -0
  14. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/README.md +347 -0
  15. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/api_server.py +633 -0
  16. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/config.json +615 -0
  17. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/evaluate.py +375 -0
  18. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/predict.py +728 -0
  19. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/requirements.txt +9 -0
  20. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/train_traditional.py +894 -0
  21. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/utils/__init__.py +0 -0
  22. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/utils/preprocessing.py +229 -0
  23. backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/visualize.py +944 -0
  24. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/Dockerfile +12 -0
  25. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/README.md +347 -0
  26. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/api_server.py +633 -0
  27. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/config.json +615 -0
  28. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/evaluate.py +375 -0
  29. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/predict.py +728 -0
  30. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/requirements.txt +9 -0
  31. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/train_traditional.py +894 -0
  32. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/utils/__init__.py +0 -0
  33. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/utils/preprocessing.py +229 -0
  34. backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/visualize.py +944 -0
  35. backend/__init__.py +0 -0
  36. backend/agents/__init__.py +104 -0
  37. backend/agents/adaptive_rag.py +470 -0
  38. backend/agents/advanced_charts.py +676 -0
  39. backend/agents/advanced_report_agent.py +0 -0
  40. backend/agents/ai_models.py +112 -0
  41. backend/agents/answer_chart_extractor.py +383 -0
  42. backend/agents/autonomous_charts.py +400 -0
  43. backend/agents/autopilot.py +830 -0
  44. backend/agents/base.py +284 -0
  45. backend/agents/base/agent_runner.py +256 -0
  46. backend/agents/causal_simulator.py +106 -0
  47. backend/agents/chart_gatekeeper.py +214 -0
  48. backend/agents/chatgpt_handler.py +240 -0
  49. backend/agents/comprehensive_charts.py +706 -0
  50. backend/agents/confidence_scorer.py +469 -0
.env.example ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============================================================================
2
+ # DATAVISION AI - ENVIRONMENT CONFIGURATION TEMPLATE
3
+ # ============================================================================
4
+ #
5
+ # COPY THIS FILE TO .env FOR LOCAL DEVELOPMENT
6
+ # DO NOT COMMIT YOUR ACTUAL .env FILE!
7
+ #
8
+ # FOR PRODUCTION (HuggingFace Spaces):
9
+ # 1. Go to Space Settings > Variables
10
+ # 2. Add each variable as a "Secret"
11
+ # ============================================================================
12
+
13
+ # ----------------------------------------------------------------------------
14
+ # ENVIRONMENT (development / production)
15
+ # ----------------------------------------------------------------------------
16
+ ENVIRONMENT=development
17
+
18
+ # ----------------------------------------------------------------------------
19
+ # AI/LLM PROVIDER (REQUIRED)
20
+ # ----------------------------------------------------------------------------
21
+ # GROQ (Primary - FREE, Fast, Reliable)
22
+ # Get from: https://console.groq.com/keys
23
+ GROQ_API_KEY=gsk_your_groq_api_key_here
24
+
25
+ # Multiple API Keys for Load Balancing (Recommended for Production)
26
+ # Add 3-5 keys to handle high traffic without rate limits
27
+ GROQ_KEY_1=gsk_backup_key_1
28
+ GROQ_KEY_2=gsk_backup_key_2
29
+ GROQ_KEY_3=gsk_backup_key_3
30
+
31
+ # Optional Fallback Providers
32
+ OPENAI_API_KEY=
33
+ GOOGLE_API_KEY=
34
+ ANTHROPIC_API_KEY=
35
+
36
+ # ----------------------------------------------------------------------------
37
+ # DATABASE & AUTHENTICATION (REQUIRED FOR PERSISTENCE)
38
+ # ----------------------------------------------------------------------------
39
+ # PostgreSQL Database Connection URL (e.g., Neon, Railway, or local Docker)
40
+ # Example: postgresql+asyncpg://user:password@hostname:5432/dbname
41
+ DATABASE_URL=postgresql+asyncpg://datavision:datavision_dev@localhost:5433/datavision
42
+
43
+ # JWT signing secret (KEEP SECRET - generate a random string)
44
+ JWT_SECRET=your_jwt_secret_here
45
+
46
+ # ----------------------------------------------------------------------------
47
+ # REDIS / RATE LIMITING (OPTIONAL)
48
+ # ----------------------------------------------------------------------------
49
+ # For Redis-backed rate limiting. If empty, uses in-memory.
50
+ REDIS_URL=
51
+
52
+ # ----------------------------------------------------------------------------
53
+ # CORS & SECURITY
54
+ # ----------------------------------------------------------------------------
55
+ CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173
56
+ ALLOWED_HOSTS=localhost,127.0.0.1
57
+
58
+ # ----------------------------------------------------------------------------
59
+ # OPTIONAL INTEGRATIONS
60
+ # ----------------------------------------------------------------------------
61
+ # Email Service (for automated reports via Resend or SMTP)
62
+ RESEND_API_KEY=
63
+ FROM_EMAIL=insights@yourdomain.com
64
+
65
+ # JWT Token Expiry (optional — defaults shown)
66
+ # ACCESS_TOKEN_EXPIRE_MINUTES=30
67
+ # REFRESH_TOKEN_EXPIRE_DAYS=7
68
+
69
+ # OAuth Providers (optional — for social login)
70
+ # GOOGLE_CLIENT_ID=
71
+ # GOOGLE_CLIENT_SECRET=
72
+ # GITHUB_CLIENT_ID=
73
+ # GITHUB_CLIENT_SECRET=
.env.production ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============================================================================
2
+ # DATAVISION AI - PRODUCTION ENVIRONMENT CONFIGURATION
3
+ # ============================================================================
4
+ #
5
+ # 🚨 IMPORTANT: Set these in HuggingFace Spaces Settings > Variables > Secrets
6
+ # DO NOT commit this file with actual keys!
7
+ #
8
+ # ============================================================================
9
+
10
+ # ----------------------------------------------------------------------------
11
+ # PRODUCTION ENVIRONMENT (CRITICAL)
12
+ # ----------------------------------------------------------------------------
13
+ ENVIRONMENT=production
14
+
15
+ # ----------------------------------------------------------------------------
16
+ # AI/LLM PROVIDER (REQUIRED)
17
+ # ----------------------------------------------------------------------------
18
+ # GROQ (Primary - FREE, Fast, Reliable)
19
+ # Get from: https://console.groq.com/keys
20
+ GROQ_API_KEY=gsk_your_groq_api_key_here
21
+
22
+ # Multiple API Keys for Load Balancing (Recommended for Production)
23
+ # Add 3-5 keys to handle high traffic without rate limits
24
+ GROQ_KEY_1=gsk_backup_key_1
25
+ GROQ_KEY_2=gsk_backup_key_2
26
+ GROQ_KEY_3=gsk_backup_key_3
27
+
28
+ # Optional Fallback Providers
29
+ OPENAI_API_KEY=sk-your_openai_key_here
30
+ GOOGLE_API_KEY=your_google_api_key_here
31
+ ANTHROPIC_API_KEY=sk-ant-your_anthropic_key_here
32
+
33
+ # ----------------------------------------------------------------------------
34
+ # DATABASE & AUTHENTICATION (REQUIRED FOR PERSISTENCE)
35
+ # ----------------------------------------------------------------------------
36
+ # PostgreSQL Database Connection URL (e.g., Neon, Railway, Supabase Postgres)
37
+ # Example: postgresql+asyncpg://user:password@hostname:5432/dbname
38
+ # Localhost will NOT work in HuggingFace Spaces.
39
+ DATABASE_URL=postgresql+asyncpg://postgres:your_password@your_db_host:5432/datavision
40
+
41
+ # JWT signing secret (KEEP SECRET - backend only)
42
+ JWT_SECRET=your_jwt_secret_here
43
+
44
+ # Redis URL (Optional for Rate Limiting in Prod)
45
+ # REDIS_URL=redis://your_redis_host:6379
46
+
47
+ # ----------------------------------------------------------------------------
48
+ # CORS & SECURITY (AUTO-CONFIGURED FOR HUGGINGFACE)
49
+ # ----------------------------------------------------------------------------
50
+ # These are automatically configured for HuggingFace Spaces
51
+ # Only change if deploying elsewhere
52
+ CORS_ORIGINS=*
53
+ ALLOWED_HOSTS=*.hf.space,huggingface.co,localhost
54
+
55
+ # ----------------------------------------------------------------------------
56
+ # OPTIONAL INTEGRATIONS
57
+ # ----------------------------------------------------------------------------
58
+ # Email Service (for automated reports via Resend)
59
+ RESEND_API_KEY=
60
+ FROM_EMAIL=insights@ai20insights.tech
61
+
62
+ # ============================================================================
63
+ # HOW TO USE IN HUGGINGFACE SPACES:
64
+ # ============================================================================
65
+ # 1. Go to your Space Settings
66
+ # 2. Click "Variables" tab
67
+ # 3. Add each variable above (without the =value part) as a "Secret"
68
+ # 4. Save and restart Space
69
+ #
70
+ # Required Variables (Minimum to run):
71
+ # - GROQ_API_KEY
72
+ # - DATABASE_URL
73
+ # - JWT_SECRET
74
+ # - ENVIRONMENT=production
75
+ #
76
+ # Recommended for Production:
77
+ # - Add GROQ_KEY_1, GROQ_KEY_2, GROQ_KEY_3 for load balancing
78
+ # - This prevents "API limit exceeded" from showing to users
79
+ # ============================================================================
.gitattributes ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Text file handling
2
+ *.md text
3
+ *.py text
4
+ *.ts text
5
+ *.tsx text
6
+ *.js text
7
+ *.json text
8
+ *.css text
9
+ *.html text
10
+
11
+ # Image & Binary files tracked by Git LFS for Hugging Face Spaces
12
+ *.png filter=lfs diff=lfs merge=lfs -text
13
+ *.jpg filter=lfs diff=lfs merge=lfs -text
14
+ *.jpeg filter=lfs diff=lfs merge=lfs -text
15
+ *.gif filter=lfs diff=lfs merge=lfs -text
16
+ *.ico filter=lfs diff=lfs merge=lfs -text
17
+ *.pdf filter=lfs diff=lfs merge=lfs -text
18
+ *.docx filter=lfs diff=lfs merge=lfs -text
.github/workflows/ci.yml ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI Pipeline
2
+
3
+ on:
4
+ push:
5
+ branches: [ "main" ]
6
+ pull_request:
7
+ branches: [ "main" ]
8
+
9
+ jobs:
10
+ backend-lint:
11
+ name: Backend Checks (Lint)
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+
17
+ - name: Set up Python
18
+ uses: actions/setup-python@v5
19
+ with:
20
+ python-version: "3.11"
21
+ cache: "pip"
22
+
23
+ - name: Install dependencies
24
+ run: |
25
+ python -m pip install --upgrade pip
26
+ pip install ruff
27
+
28
+ - name: Lint with Ruff
29
+ run: |
30
+ ruff check backend/ --select=E9,F63,F7,F82
31
+
32
+ frontend-build-and-deploy:
33
+ name: Build & Deploy to HuggingFace
34
+ needs: backend-lint
35
+ runs-on: ubuntu-latest
36
+ if: github.ref == 'refs/heads/main'
37
+
38
+ steps:
39
+ - uses: actions/checkout@v4
40
+ with:
41
+ fetch-depth: 0
42
+ lfs: true
43
+
44
+ - name: Setup Node.js
45
+ uses: actions/setup-node@v4
46
+ with:
47
+ node-version: "20"
48
+
49
+ - name: Install Frontend Dependencies
50
+ working-directory: ./frontend
51
+ run: npm install
52
+
53
+ - name: Build Frontend
54
+ working-directory: ./frontend
55
+ run: npm run build
56
+
57
+ - name: Push Clean Release to Hugging Face Space
58
+ env:
59
+ HF_TOKEN: ${{ secrets.HF_TOKEN }}
60
+ HUGGINGFACE_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
61
+ HF_SPACE: ${{ secrets.HF_SPACE }}
62
+ run: |
63
+ TOKEN="${HF_TOKEN:-$HUGGINGFACE_TOKEN}"
64
+ SPACE_NAME="${HF_SPACE:-datavision-ai/Datavision}"
65
+
66
+ if [ -n "$TOKEN" ]; then
67
+ echo "🚀 Syncing clean release to Hugging Face Space: $SPACE_NAME"
68
+ git config --global user.email "ci@datavision.ai"
69
+ git config --global user.name "DataVision CI/CD Bot"
70
+ git config --global lfs.allowincompletepush false
71
+ git lfs install
72
+ git lfs pull
73
+
74
+ # Create clean orphan release branch without historical broken LFS pointers
75
+ git checkout --orphan hf-release
76
+ git reset
77
+
78
+ # Stage current active files and built frontend assets
79
+ git add -A
80
+ git add -f frontend/dist 2>/dev/null || true
81
+ git commit -m "release: clean production build for HuggingFace Space" || true
82
+
83
+ # Setup HF remote and force push clean orphan branch to main
84
+ git remote remove hf 2>/dev/null || true
85
+ git remote add hf "https://user:$TOKEN@huggingface.co/spaces/$SPACE_NAME"
86
+ git lfs push hf hf-release
87
+ git push -f hf hf-release:main
88
+ echo "✅ Successfully pushed clean release to Hugging Face Space: $SPACE_NAME"
89
+ else
90
+ echo "⚠️ Secret HF_TOKEN not set. Set HF_TOKEN in GitHub Settings -> Secrets -> Actions to auto-deploy."
91
+ fi
.gitignore ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+
12
+ # Node
13
+ node_modules/
14
+ npm-debug.log*
15
+ yarn-debug.log*
16
+ yarn-error.log*
17
+
18
+ # Environment (CRITICAL: never commit secrets)
19
+ .env
20
+ .env.local
21
+ .env.*.local
22
+ backend/.env
23
+ **/.env
24
+
25
+ # SQLite databases (user data)
26
+ *.db
27
+ *.sqlite3
28
+
29
+ # IDE
30
+ .vscode/
31
+ .idea/
32
+ *.swp
33
+ *.swo
34
+ *~
35
+
36
+ # OS
37
+ .DS_Store
38
+ Thumbs.db
39
+
40
+ # Storage (keep structure, ignore user data)
41
+ storage/
42
+ backend/storage/
43
+
44
+ # Debug artifacts
45
+ response.json
46
+ search_out*.txt
47
+ backend/response.json
48
+ backend/search_out*.txt
49
+
50
+ # Logs
51
+ *.log
52
+ logs/
53
+
54
+ # CatBoost
55
+ catboost_info/
56
+ backend/catboost_info/
57
+
58
+ # Cache
59
+ .cache/
60
+ *.cache
61
+ .pytest_cache/
62
+ .mypy_cache/
63
+ .ruff_cache/
64
+
65
+ # Build output (tracked for Hugging Face Space Docker build)
66
+ frontend/dist/**/*.map
67
+
68
+ # Temporary
69
+ *.tmp
70
+ *.temp
71
+
72
+ # Lock files
73
+ uv.lock
74
+
75
+ # Model files
76
+ *.bin
77
+ *.ckpt
78
+ *.pth
79
+ *.safetensors
80
+ *.onnx
81
+
82
+ # Test output files
83
+ *_results.txt
84
+ *_output.txt
85
+ models_list.txt
86
+ push_log.txt
87
+
88
+ # Large files
89
+ *.zip
90
+ *.tar.gz
91
+
92
+ # Sample data (not needed for production)
93
+ sample_data/
94
+
95
+ # Backend uploads (user data)
96
+ backend/uploads/
97
+
98
+ # FAISS indexes
99
+ *.faiss
100
+ faiss/
101
+
102
+ # Qdrant data
103
+ qdrant_data/
104
+
105
+ # Computer Vision model weights & dataset caches
106
+ temp_hf_space/
107
+ backend/cv_datasets/
108
+ backend/cv_models/
109
+ backend/sample_cv_test_datasets/
110
+ backend/runs/
111
+ backend/*.pt
112
+ *.pt
.python-version ADDED
@@ -0,0 +1 @@
 
 
1
+ 3.12
AUTOML_DOCUMENTATION.md ADDED
@@ -0,0 +1,348 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚀 DataVision AutoML - Complete Documentation
2
+
3
+ ## Table of Contents
4
+ 1. [Overview](#overview)
5
+ 2. [Training Modes](#training-modes)
6
+ 3. [Supervised Learning Algorithms](#supervised-learning-algorithms)
7
+ 4. [Unsupervised Learning Algorithms](#unsupervised-learning-algorithms)
8
+ 5. [NLP Algorithms](#nlp-algorithms)
9
+ 6. [Deep Learning Architectures](#deep-learning-architectures)
10
+ 7. [Charts & Visualizations](#charts--visualizations)
11
+ 8. [API Endpoints](#api-endpoints)
12
+
13
+ ---
14
+
15
+ ## Overview
16
+
17
+ DataVision AutoML is a production-grade machine learning platform supporting:
18
+ - **30+ ML Algorithms** (Classification, Regression, Clustering)
19
+ - **Auto GPU/CPU Detection** (CUDA, ROCm, Metal)
20
+ - **Advanced Data Cleaning** (Smart Imputation, Outlier Detection)
21
+ - **Feature Selection** (Variance, Correlation, Mutual Info, RFE)
22
+ - **Bayesian Hyperparameter Optimization** (Optuna)
23
+ - **Ensemble Methods** (Stacking, Voting, Blending)
24
+ - **20+ Visualization Charts**
25
+
26
+ ---
27
+
28
+ ## Training Modes
29
+
30
+ ### 1. Fast Mode (Default)
31
+ - **Speed**: ~30-60 seconds
32
+ - **Algorithms**: Top 5 performing algorithms
33
+ - **CV Folds**: 3-fold cross-validation
34
+ - **Use Case**: Quick prototyping, small datasets
35
+
36
+ ### 2. Ultra Mode
37
+ - **Speed**: 2-10 minutes
38
+ - **Algorithms**: All 15+ algorithms
39
+ - **CV Folds**: 5-fold stratified cross-validation
40
+ - **Features**: Hyperparameter tuning, ensemble stacking
41
+ - **Use Case**: Production models, maximum accuracy
42
+
43
+ ### 3. Multi-Mode Training
44
+ - **Combines**: Traditional ML + NLP + Deep Learning
45
+ - **Parallel Training**: Runs selected modes simultaneously
46
+ - **Best Model Selection**: Cross-mode comparison
47
+
48
+ ---
49
+
50
+ ## Supervised Learning Algorithms
51
+
52
+ ### Classification Algorithms
53
+
54
+ | Algorithm | Parameters | Use Case | Chart Support |
55
+ |-----------|-----------|----------|---------------|
56
+ | **XGBoost** | `n_estimators=100-500`, `max_depth=3-10`, `learning_rate=0.01-0.3`, `subsample=0.6-1.0`, `colsample_bytree=0.6-1.0`, `min_child_weight=1-10`, `gamma=0-5`, `reg_alpha=0-1`, `reg_lambda=0-1` | Large datasets, tabular data | ✅ All |
57
+ | **LightGBM** | `n_estimators=100-500`, `max_depth=3-15`, `learning_rate=0.01-0.3`, `num_leaves=20-150`, `subsample=0.6-1.0`, `colsample_bytree=0.6-1.0`, `min_child_samples=10-100`, `reg_alpha=0-1`, `reg_lambda=0-1` | Fast training, large data | ✅ All |
58
+ | **CatBoost** | `iterations=100-500`, `depth=4-10`, `learning_rate=0.01-0.3`, `l2_leaf_reg=1-10`, `border_count=32-255`, `random_strength=0-10` | Categorical features, GPU | ✅ All |
59
+ | **Random Forest** | `n_estimators=100-500`, `max_depth=5-30`, `min_samples_split=2-20`, `min_samples_leaf=1-10`, `max_features='sqrt'/'log2'/0.5-1.0`, `bootstrap=True/False` | General purpose, interpretable | ✅ All |
60
+ | **Gradient Boosting** | `n_estimators=100-300`, `max_depth=3-8`, `learning_rate=0.01-0.2`, `subsample=0.6-1.0`, `min_samples_split=2-20`, `min_samples_leaf=1-10` | Balanced performance | ✅ All |
61
+ | **Extra Trees** | `n_estimators=100-500`, `max_depth=5-30`, `min_samples_split=2-20`, `min_samples_leaf=1-10`, `max_features='sqrt'/'log2'/0.5-1.0` | Fast random splits | ✅ All |
62
+ | **Hist Gradient Boosting** | `max_iter=100-500`, `max_depth=3-15`, `learning_rate=0.01-0.3`, `min_samples_leaf=10-50`, `l2_regularization=0-1` | Fast, native categorical | ✅ All |
63
+ | **AdaBoost** | `n_estimators=50-200`, `learning_rate=0.01-1.0`, `algorithm='SAMME'/'SAMME.R'` | Boosting weak learners | ✅ All |
64
+ | **Bagging** | `n_estimators=10-50`, `max_samples=0.5-1.0`, `max_features=0.5-1.0`, `bootstrap=True/False` | Variance reduction | ✅ All |
65
+ | **Logistic Regression** | `C=0.001-100`, `penalty='l1'/'l2'/'elasticnet'`, `solver='lbfgs'/'saga'/'liblinear'`, `max_iter=100-1000`, `l1_ratio=0-1` | Linear baseline, interpretable | ✅ All |
66
+ | **SVC (SVM)** | `C=0.1-100`, `kernel='rbf'/'linear'/'poly'/'sigmoid'`, `gamma='scale'/'auto'/0.001-10`, `degree=2-5`, `class_weight='balanced'/None` | Non-linear boundaries | ✅ All |
67
+ | **Linear SVC** | `C=0.1-100`, `penalty='l1'/'l2'`, `loss='hinge'/'squared_hinge'`, `max_iter=1000-10000` | Large-scale linear | ✅ All |
68
+ | **K-Neighbors** | `n_neighbors=3-15`, `weights='uniform'/'distance'`, `metric='euclidean'/'manhattan'/'minkowski'`, `p=1-2`, `leaf_size=20-50` | Instance-based | ✅ All |
69
+ | **Gaussian NB** | `var_smoothing=1e-9-1e-6` | Text, probabilistic | ✅ All |
70
+ | **Multinomial NB** | `alpha=0.01-1.0`, `fit_prior=True/False` | Text classification | ✅ All |
71
+ | **Complement NB** | `alpha=0.01-1.0`, `norm=True/False` | Imbalanced text | ✅ All |
72
+ | **Bernoulli NB** | `alpha=0.01-1.0`, `binarize=0.0-1.0` | Binary features | ✅ All |
73
+ | **MLP Classifier** | `hidden_layer_sizes=(64,32)-(256,128,64)`, `activation='relu'/'tanh'`, `solver='adam'/'sgd'`, `alpha=0.0001-0.01`, `learning_rate='constant'/'adaptive'`, `max_iter=200-1000`, `early_stopping=True` | Neural network | ✅ All |
74
+ | **Decision Tree** | `max_depth=3-20`, `min_samples_split=2-20`, `min_samples_leaf=1-10`, `criterion='gini'/'entropy'`, `splitter='best'/'random'` | Interpretable rules | ✅ All |
75
+ | **LDA** | `solver='svd'/'lsqr'/'eigen'`, `shrinkage='auto'/0-1` | Dimensionality reduction | ✅ All |
76
+ | **QDA** | `reg_param=0-1` | Quadratic boundaries | ✅ All |
77
+ | **SGD Classifier** | `loss='hinge'/'log_loss'/'perceptron'`, `penalty='l1'/'l2'/'elasticnet'`, `alpha=0.0001-0.01`, `max_iter=1000-5000`, `learning_rate='optimal'/'constant'/'adaptive'` | Large-scale online | ✅ All |
78
+ | **Passive Aggressive** | `C=0.1-10`, `max_iter=1000-5000`, `loss='hinge'/'squared_hinge'` | Online learning | ✅ All |
79
+
80
+ ### Regression Algorithms
81
+
82
+ | Algorithm | Parameters | Use Case |
83
+ |-----------|-----------|----------|
84
+ | **XGBoost Regressor** | Same as classifier | General regression |
85
+ | **LightGBM Regressor** | Same as classifier | Fast regression |
86
+ | **CatBoost Regressor** | Same as classifier | Categorical features |
87
+ | **Random Forest Regressor** | Same as classifier | General purpose |
88
+ | **Gradient Boosting Regressor** | Same as classifier | Balanced |
89
+ | **Extra Trees Regressor** | Same as classifier | Fast |
90
+ | **Ridge** | `alpha=0.01-100`, `solver='auto'/'svd'/'cholesky'/'lsqr'` | L2 regularization |
91
+ | **Lasso** | `alpha=0.01-100`, `max_iter=1000-10000` | L1 regularization, feature selection |
92
+ | **ElasticNet** | `alpha=0.01-100`, `l1_ratio=0-1`, `max_iter=1000-10000` | Combined L1+L2 |
93
+ | **SVR** | `C=0.1-100`, `kernel='rbf'/'linear'/'poly'`, `epsilon=0.01-0.5`, `gamma='scale'/'auto'` | Non-linear |
94
+ | **Linear SVR** | `C=0.1-100`, `epsilon=0-0.5`, `max_iter=1000-10000` | Large-scale linear |
95
+ | **K-Neighbors Regressor** | Same as classifier | Instance-based |
96
+ | **MLP Regressor** | Same as classifier | Neural network |
97
+ | **Bayesian Ridge** | `alpha_1=1e-6`, `alpha_2=1e-6`, `lambda_1=1e-6`, `lambda_2=1e-6` | Probabilistic |
98
+ | **Huber Regressor** | `epsilon=1.0-2.0`, `alpha=0.0001-0.01`, `max_iter=100-500` | Robust to outliers |
99
+ | **Poisson Regressor** | `alpha=0-1`, `max_iter=100-500` | Count data |
100
+ | **Quantile Regressor** | `quantile=0.1-0.9`, `alpha=0-1` | Quantile estimation |
101
+ | **Lasso Lars** | `alpha=0.01-100`, `max_iter=500-1000` | Feature selection |
102
+
103
+ ---
104
+
105
+ ## Unsupervised Learning Algorithms
106
+
107
+ ### Clustering Algorithms
108
+
109
+ | Algorithm | Parameters | Charts Generated | Use Case |
110
+ |-----------|-----------|------------------|----------|
111
+ | **KMeans** | `n_clusters=2-15` (auto-detect), `init='k-means++'/'random'`, `n_init=10-30`, `max_iter=300-500`, `tol=1e-4`, `algorithm='lloyd'/'elkan'` | Elbow, Silhouette, Scatter, Distribution, Heatmap, PCA, 3D, t-SNE, UMAP, Pairplot, Boxplot, Violin, Correlation, Radar | General clustering |
112
+ | **DBSCAN** | `eps=auto-detect` (k-distance), `min_samples=max(3, n//100)` (auto), `metric='euclidean'/'manhattan'`, `algorithm='auto'/'ball_tree'/'kd_tree'` | K-Distance, Scatter, Distribution, Silhouette, Heatmap, PCA, 3D, t-SNE, UMAP, Pairplot, Boxplot, Violin, Correlation, Radar | Noise detection, arbitrary shapes |
113
+ | **Hierarchical (Agglomerative)** | `n_clusters=2-15`, `linkage='ward'/'average'/'complete'/'single'`, `metric='euclidean'`, `compute_full_tree='auto'` | **Dendrogram**, Silhouette, Scatter, Distribution, Heatmap, PCA, 3D, t-SNE, UMAP, Pairplot, Boxplot, Violin, Correlation, Radar | Hierarchical structure |
114
+ | **GMM (Gaussian Mixture)** | `n_components=2-15`, `covariance_type='full'/'tied'/'diag'/'spherical'`, `n_init=10`, `max_iter=200`, `init_params='k-means++'` | **BIC/AIC Chart**, Silhouette, Scatter, Distribution, Heatmap, PCA, 3D, t-SNE, UMAP, Pairplot, Boxplot, Violin, Correlation, Radar | Probabilistic clustering |
115
+ | **Spectral** | `n_clusters=2-15`, `affinity='nearest_neighbors'/'rbf'`, `n_neighbors=10-30`, `assign_labels='cluster_qr'/'kmeans'`, `gamma=1.0` | **Affinity Matrix**, Silhouette, Scatter, Distribution, Heatmap, PCA, 3D, t-SNE, UMAP, Pairplot, Boxplot, Violin, Correlation, Radar | Graph-based clustering |
116
+
117
+ ### Clustering Charts (17 Total)
118
+
119
+ | Chart | Description | Size (figsize) | When Generated |
120
+ |-------|-------------|----------------|----------------|
121
+ | `cluster_scatter` | PCA 2D scatter with centroids | (10, 8) | Always |
122
+ | `elbow_method` | Silhouette scores vs k | (10, 6) | KMeans/Hierarchical/GMM/Spectral |
123
+ | `silhouette_plot` | Per-sample silhouette coefficients | (10, 8) | n_clusters≥2, n_samples≥50 |
124
+ | `cluster_distribution` | Bar chart of cluster sizes | (10, 6) | Always |
125
+ | `cluster_heatmap` | Feature means per cluster | (12, 8) | n_features≥3, n_clusters≥2 |
126
+ | `pca_variance` | Explained variance per component | (14, 5) | n_features≥3 |
127
+ | `cluster_3d` | PCA 3D scatter | (12, 10) | n_features≥3, n_samples≥50 |
128
+ | `dendrogram` | Hierarchical tree structure | (14, 8) | **Hierarchical only** |
129
+ | `gmm_bic_aic` | BIC/AIC model selection | (10, 6) | **GMM only** |
130
+ | `dbscan_kdist` | K-distance graph for eps | (10, 6) | **DBSCAN only** |
131
+ | `spectral_affinity` | Affinity matrix heatmap | (10, 8) | **Spectral only**, n_samples≤500 |
132
+ | `tsne` | t-SNE 2D visualization | (10, 8) | 100≤n_samples≤3000, n_features≥3 |
133
+ | `umap` | UMAP 2D visualization | (10, 8) | 100≤n_samples≤5000, n_features≥3 |
134
+ | `pairplot` | Feature pairwise scatter | auto | 2≤n_features≤6, n_samples≤2000 |
135
+ | `boxplots` | Feature distribution by cluster | (4*n_features, 6) | n_features≥2, n_clusters≥2 |
136
+ | `violin_plots` | Violin distribution by cluster | (5*n_features, 6) | n_features≥2, n_clusters≥2, n_samples≥50 |
137
+ | `correlation_heatmap` | Feature correlation matrix | (10, 8) | n_features≥3 |
138
+ | `radar_chart` | Cluster comparison radar | (10, 10) | n_features≥3, n_clusters≥2 |
139
+
140
+ ### Clustering Metrics
141
+
142
+ | Metric | Range | Description |
143
+ |--------|-------|-------------|
144
+ | `silhouette_score` | [-1, 1] | Higher is better, cluster cohesion vs separation |
145
+ | `calinski_harabasz_score` | [0, ∞) | Higher is better, variance ratio |
146
+ | `davies_bouldin_score` | [0, ∞) | Lower is better, average similarity |
147
+ | `reliability_score` | [0, 100] | Production intelligence quality score |
148
+
149
+ ---
150
+
151
+ ## NLP Algorithms
152
+
153
+ ### Text Classification Algorithms
154
+
155
+ | Algorithm | Parameters | Vectorizer | Use Case |
156
+ |-----------|-----------|------------|----------|
157
+ | **TF-IDF + Logistic Regression** | `C=0.1-100`, `max_iter=500` | TF-IDF: `max_features=5000-20000`, `ngram_range=(1,2)/(1,3)`, `min_df=2-5`, `max_df=0.9-0.95` | General text |
158
+ | **TF-IDF + SVM** | `C=0.1-100`, `kernel='linear'/'rbf'` | Same as above | High accuracy |
159
+ | **TF-IDF + Naive Bayes** | `alpha=0.01-1.0` | Same as above | Fast, probabilistic |
160
+ | **TF-IDF + Random Forest** | `n_estimators=100-300` | Same as above | Feature importance |
161
+ | **TF-IDF + XGBoost** | Same as classifier | Same as above | Best accuracy |
162
+ | **BOW + Classifiers** | Same as TF-IDF | CountVectorizer: same params | Simple baseline |
163
+ | **N-gram Models** | Same as TF-IDF | `ngram_range=(2,3)/(1,4)` | Phrase patterns |
164
+
165
+ ### NLP-Specific Charts
166
+
167
+ | Chart | Description | When Generated |
168
+ |-------|-------------|----------------|
169
+ | `confusion_matrix` | Class prediction matrix | Always |
170
+ | `word_cloud` | Important words visualization | Text data |
171
+ | `class_distribution` | Target class frequencies | Always |
172
+ | `roc_curve` | ROC-AUC per class | Multi-class |
173
+ | `precision_recall` | PR curve per class | Multi-class |
174
+
175
+ ---
176
+
177
+ ## Deep Learning Architectures
178
+
179
+ ### MLP (Multi-Layer Perceptron) Architectures
180
+
181
+ | Architecture | Hidden Layers | Parameters | Use Case |
182
+ |--------------|---------------|-----------|----------|
183
+ | **mlp_small** | (64, 32) | ~5K params | Small data, fast |
184
+ | **mlp_medium** | (128, 64, 32) | ~15K params | Balanced |
185
+ | **mlp_large** | (256, 128, 64) | ~50K params | Complex patterns |
186
+ | **mlp_wide** | (512, 256) | ~150K params | High-dimensional |
187
+ | **mlp_deep** | (128, 128, 128, 128) | ~70K params | Deep representation |
188
+
189
+ ### MLP Training Parameters
190
+
191
+ | Parameter | Values | Description |
192
+ |-----------|--------|-------------|
193
+ | `activation` | 'relu', 'tanh', 'logistic' | Activation function |
194
+ | `solver` | 'adam', 'sgd', 'lbfgs' | Optimizer |
195
+ | `alpha` | 0.0001-0.01 | L2 regularization |
196
+ | `learning_rate` | 'constant', 'adaptive', 'invscaling' | LR schedule |
197
+ | `learning_rate_init` | 0.001-0.01 | Initial LR |
198
+ | `max_iter` | 200-1000 | Max epochs |
199
+ | `early_stopping` | True | Validation-based stopping |
200
+ | `validation_fraction` | 0.1 | Validation split |
201
+ | `n_iter_no_change` | 10 | Early stopping patience |
202
+ | `batch_size` | 32-256, 'auto' | Mini-batch size |
203
+
204
+ ### Deep Learning Charts
205
+
206
+ | Chart | Description | When Generated |
207
+ |-------|-------------|----------------|
208
+ | `learning_curve` | Loss vs epochs | Training history |
209
+ | `confusion_matrix` | Prediction matrix | Classification |
210
+ | `feature_importance` | Input weights | All |
211
+ | `activation_distribution` | Layer activations | Debug |
212
+
213
+ ---
214
+
215
+ ## Charts & Visualizations
216
+
217
+ ### Supervised Learning Charts (Classification)
218
+
219
+ | Chart Key | Description | Size | Parameters |
220
+ |-----------|-------------|------|------------|
221
+ | `confusion_matrix` | True vs Predicted heatmap | (10, 8) | `cmap='Blues'`, `annot=True`, `fmt='d'` |
222
+ | `feature_importance` | Top 15 features bar chart | (12, 8) | Sorted descending |
223
+ | `roc_curve` | ROC curves per class | (10, 8) | `lw=2`, AUC in legend |
224
+ | `precision_recall` | PR curves per class | (10, 8) | AP in legend |
225
+ | `class_distribution` | Target class bar chart | (10, 6) | With percentages |
226
+ | `learning_curve` | Train/Val score vs size | (10, 6) | CV=5 |
227
+ | `calibration_curve` | Predicted vs actual prob | (10, 8) | Perfectly calibrated line |
228
+ | `lift_curve` | Cumulative gains | (10, 8) | Baseline comparison |
229
+ | `correlation_heatmap` | Feature correlations | (12, 10) | `cmap='coolwarm'` |
230
+ | `shap_summary` | SHAP feature values | (12, 10) | Requires SHAP |
231
+
232
+ ### Supervised Learning Charts (Regression)
233
+
234
+ | Chart Key | Description | Size | Parameters |
235
+ |-----------|-------------|------|------------|
236
+ | `actual_vs_predicted` | Scatter with ideal line | (10, 8) | `alpha=0.5` |
237
+ | `residual_plot` | Residuals vs predicted | (10, 6) | Zero line |
238
+ | `residual_distribution` | Residual histogram+KDE | (10, 6) | Normal curve overlay |
239
+ | `feature_importance` | Top 15 features | (12, 8) | Sorted descending |
240
+ | `error_by_range` | MAE per target bin | (10, 6) | 10 bins |
241
+ | `qq_plot` | Quantile-quantile | (8, 8) | Normal reference |
242
+ | `learning_curve` | Train/Val RMSE vs size | (10, 6) | CV=5 |
243
+ | `cook_distance` | Influential points | (10, 6) | Threshold line |
244
+
245
+ ### Chart Generation Parameters
246
+
247
+ ```python
248
+ # Common matplotlib settings
249
+ plt.style.use('seaborn-v0_8-whitegrid')
250
+ plt.rcParams['figure.dpi'] = 150
251
+ plt.rcParams['savefig.dpi'] = 150
252
+ plt.rcParams['savefig.bbox'] = 'tight'
253
+ plt.rcParams['savefig.facecolor'] = 'white'
254
+
255
+ # Color palettes
256
+ CLASSIFICATION_COLORS = ['#2563eb', '#16a34a', '#dc2626', '#ea580c', '#9333ea',
257
+ '#0891b2', '#db2777', '#d97706', '#0d9488', '#4f46e5']
258
+ REGRESSION_COLORS = ['#3b82f6', '#ef4444']
259
+ CLUSTER_COLORS = ['#2563eb', '#16a34a', '#dc2626', '#ea580c', '#9333ea',
260
+ '#0891b2', '#db2777', '#d97706', '#0d9488', '#4f46e5',
261
+ '#84cc16', '#06b6d4', '#f43f5e', '#8b5cf6', '#14b8a6']
262
+ ```
263
+
264
+ ---
265
+
266
+ ## API Endpoints
267
+
268
+ ### Supervised Learning
269
+
270
+ | Endpoint | Method | Mode | Description |
271
+ |----------|--------|------|-------------|
272
+ | `/api/v1/automl/production_train` | POST | Fast | Quick training with top algorithms |
273
+ | `/api/v1/automl/train` | POST | Standard | Full training pipeline |
274
+ | `/api/v1/automl/ultra_train` | POST | Ultra | Maximum accuracy mode |
275
+ | `/api/v1/automl/multi_mode/train` | POST | Multi | Combined Traditional+NLP+Deep |
276
+ | `/api/v1/automl/predict` | POST | - | Single prediction |
277
+ | `/api/v1/automl/batch_predict` | POST | - | Batch predictions |
278
+ | `/api/v1/automl/saved-result` | GET | - | Load saved model results |
279
+ | `/api/v1/automl/stop_training` | POST | - | Cancel ongoing training |
280
+
281
+ ### NLP
282
+
283
+ | Endpoint | Method | Description |
284
+ |----------|--------|-------------|
285
+ | `/api/v1/automl/nlp/train` | POST | Text classification training |
286
+ | `/api/v1/automl/nlp/predict` | POST | Text prediction |
287
+
288
+ ### Deep Learning
289
+
290
+ | Endpoint | Method | Description |
291
+ |----------|--------|-------------|
292
+ | `/api/v1/automl/deep_learning/train` | POST | Neural network training |
293
+ | `/api/v1/automl/deep_learning/predict` | POST | Deep learning prediction |
294
+
295
+ ### Unsupervised Learning
296
+
297
+ | Endpoint | Method | Description |
298
+ |----------|--------|-------------|
299
+ | `/api/v1/ml/clustering` | POST | Clustering analysis |
300
+ | `/api/v1/ml/clustering/predict` | POST | Predict cluster for new data |
301
+ | `/api/v1/ml/clustering/download-model/{user_id}` | GET | Download PKL model |
302
+ | `/api/v1/ml/clustering/download-data/{user_id}` | GET | Download clustered CSV |
303
+
304
+ ---
305
+
306
+ ## Production Intelligence
307
+
308
+ ### Reliability Score (0-100)
309
+
310
+ Computed based on:
311
+ - Data quality (missing values, outliers)
312
+ - Model validation (CV performance, overfitting check)
313
+ - Feature quality (variance, correlation)
314
+ - Sample size adequacy
315
+
316
+ ### Validation Warnings
317
+
318
+ | Warning Type | Trigger |
319
+ |--------------|---------|
320
+ | `small_dataset` | n_samples < 100 |
321
+ | `high_cardinality` | categorical unique > 50% |
322
+ | `class_imbalance` | minority class < 10% |
323
+ | `missing_values` | missing > 20% |
324
+ | `potential_leakage` | feature corr > 0.95 with target |
325
+ | `low_variance` | feature variance ≈ 0 |
326
+
327
+ ---
328
+
329
+ ## File Outputs
330
+
331
+ ### Saved Artifacts
332
+
333
+ | File | Location | Content |
334
+ |------|----------|---------|
335
+ | `best_model.pkl` | `storage/users/{user_id}/models/` | Trained model + metadata |
336
+ | `cleaned_data.csv` | `storage/users/{user_id}/files/` | Preprocessed data |
337
+ | `multimode_metadata.json` | `storage/users/{user_id}/models/` | Training configuration |
338
+ | `clustering_model.pkl` | `storage/users/{user_id}/models/` | Clustering model + scaler |
339
+ | `clustered_data.csv` | `storage/users/{user_id}/files/` | Data with cluster labels |
340
+
341
+ ---
342
+
343
+ ## Version Info
344
+
345
+ - **Engine Version**: 7.0
346
+ - **Last Updated**: February 2026
347
+ - **Supported Python**: 3.11+
348
+ - **Key Dependencies**: scikit-learn>=1.3.0, xgboost>=2.0.0, lightgbm>=4.2.0, catboost>=1.2.0, optuna>=3.4.0
Dockerfile ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.12-slim
2
+
3
+ # DataVision AI — Production Container
4
+ # Pre-built frontend committed as static/ — no Node.js build needed.
5
+
6
+ WORKDIR /app
7
+
8
+ # Install system dependencies required by ML libraries
9
+ RUN apt-get update && apt-get install -y --no-install-recommends \
10
+ libgomp1 \
11
+ libgl1 \
12
+ libglib2.0-0 \
13
+ curl \
14
+ && rm -rf /var/lib/apt/lists/* \
15
+ && apt-get clean
16
+
17
+ # Create non-root user for security
18
+ RUN groupadd -r appuser && useradd -r -g appuser appuser
19
+
20
+ # Install Python dependencies
21
+ COPY requirements.txt .
22
+ RUN pip install --no-cache-dir -r requirements.txt
23
+
24
+ # Copy frontend public assets and pre-built dist
25
+ COPY frontend/public /app/frontend/public
26
+ COPY frontend/dist /app/static
27
+ COPY frontend/dist /app/backend/static
28
+
29
+ # Copy backend source files (includes both legacy and new app/)
30
+ COPY backend/ backend/
31
+
32
+ # Copy start script
33
+ COPY start.sh /app/
34
+ RUN chmod +x /app/start.sh
35
+
36
+ # Create storage directories with proper ownership
37
+ RUN mkdir -p /app/storage/users \
38
+ /app/backend/storage/users \
39
+ /app/backend/storage/automl \
40
+ /app/backend/storage/models \
41
+ /app/backend/storage/clustering_models \
42
+ /app/backend/storage/graph \
43
+ /app/backend/storage/faiss \
44
+ /app/backend/storage/uploads \
45
+ /app/backend/storage/exports \
46
+ /app/backend/storage/audit \
47
+ /app/backend/catboost_info && \
48
+ chown -R appuser:appuser /app/storage /app/backend/storage /app/backend/catboost_info /app/static && \
49
+ chmod -R 755 /app/storage /app/backend/storage /app/backend/catboost_info
50
+
51
+ # Set ownership of entire app directory to non-root user
52
+ RUN chown -R appuser:appuser /app
53
+
54
+ # Python path includes both backend/ (legacy) and backend/ for app.* imports
55
+ ENV PYTHONPATH=/app/backend
56
+ ENV PYTHONDONTWRITEBYTECODE=1
57
+ ENV PYTHONUNBUFFERED=1
58
+ ENV PYTHONIOENCODING=utf-8
59
+ ENV LC_ALL=C.UTF-8
60
+ ENV LANG=C.UTF-8
61
+ ENV MPLCONFIGDIR=/tmp/matplotlib
62
+ ENV HOME=/tmp
63
+
64
+ # Switch to non-root user
65
+ USER appuser
66
+
67
+ # Set working directory to backend for correct imports
68
+ WORKDIR /app/backend
69
+
70
+ # Expose port
71
+ EXPOSE 7860
72
+
73
+ # Health check
74
+ HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
75
+ CMD curl -f http://localhost:7860/health || exit 1
76
+
77
+ # Start using the start script (handles migrations)
78
+ CMD ["/app/start.sh"]
79
+
HUGGINGFACE_DEPLOYMENT.md ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚀 HuggingFace Spaces Deployment Guide
2
+
3
+ ## Production-Ready DataVision AI
4
+
5
+ This project is configured with a **fully automated CI/CD pipeline**. Any code you push to the `main` branch of this GitHub repository will automatically be deployed to your HuggingFace Space.
6
+
7
+ Your HuggingFace Space: [https://datavision-ai-datavision.hf.space](https://datavision-ai-datavision.hf.space)
8
+
9
+ ### Pre-Deployment Checklist ✅
10
+
11
+ #### 1. Setup GitHub Actions CI/CD (One-time setup)
12
+ To allow GitHub to push to HuggingFace Spaces on your behalf, you need a HuggingFace Write Token:
13
+ 1. Go to your HuggingFace Settings: [https://huggingface.co/settings/tokens](https://huggingface.co/settings/tokens)
14
+ 2. Create a new token with **Write** permissions and copy it.
15
+ 3. Go to this GitHub repository's Settings > **Secrets and variables** > **Actions**
16
+ 4. Click **New repository secret**
17
+ 5. Name: `HF_TOKEN`
18
+ 6. Secret: paste your copied HuggingFace token
19
+
20
+ #### 2. Environment Variables (CRITICAL)
21
+ Your actual secrets must be added to your HuggingFace Space (they are NOT in GitHub).
22
+ Go to HuggingFace Space Settings > **Variables and secrets**.
23
+ Add the following as **Secrets**:
24
+
25
+ ```bash
26
+ # REQUIRED - AI Provider
27
+ GROQ_API_KEY=gsk_your_actual_groq_key_here
28
+
29
+ # REQUIRED - Database (PostgreSQL hosted e.g. Neon, Supabase Postgres, Railway)
30
+ # Localhost will NOT work on HuggingFace.
31
+ DATABASE_URL=postgresql+asyncpg://user:password@hostname:5432/datavision
32
+
33
+ # REQUIRED - Authentication
34
+ JWT_SECRET=your_randomly_generated_jwt_secret
35
+
36
+ # Production Environment
37
+ ENVIRONMENT=production
38
+
39
+ # CORS (Auto-configured for HF Spaces)
40
+ CORS_ORIGINS=*
41
+ ALLOWED_HOSTS=*.hf.space,huggingface.co
42
+ ```
43
+
44
+ ---
45
+
46
+ ## Deployment Workflow
47
+
48
+ ### 1. The Automated Way (GitHub CI/CD) - Recommended
49
+ Because we've configured GitHub Actions, deployment is completely hands-off.
50
+
51
+ ```bash
52
+ # 1. Make your changes locally
53
+ git add .
54
+ git commit -m "Your descriptive commit message"
55
+
56
+ # 2. Push to GitHub
57
+ git push origin main
58
+ ```
59
+ **What happens next?**
60
+ 1. GitHub Actions automatically starts the CI Pipeline (`ci.yml`). It lints the Python backend and builds the frontend.
61
+ 2. If successful, it starts the Deployment Pipeline (`deploy.yml`).
62
+ 3. GitHub pushes the code directly to HuggingFace.
63
+ 4. HuggingFace rebuilds the Docker container automatically (takes ~5 minutes).
64
+
65
+ ### 2. Manual Fallback Way (If CI fails)
66
+ If you ever need to bypass GitHub Actions and push directly to HuggingFace:
67
+
68
+ ```bash
69
+ # 1. Add HuggingFace remote manually (first time only)
70
+ git remote add huggingface https://huggingface.co/spaces/datavision-ai/Datavision
71
+
72
+ # 2. Force push
73
+ git push --force huggingface main
74
+ ```
75
+
76
+ ---
77
+
78
+ ## Database Configuration for HuggingFace
79
+
80
+ HuggingFace Spaces are ephemeral Docker containers. This means a local SQLite or Localhost PostgreSQL database will **lose all data when the space goes to sleep**.
81
+
82
+ To maintain user accounts, chats, and configurations, you MUST use an external hosted PostgreSQL database.
83
+
84
+ **Recommended Free Providers:**
85
+ 1. **Neon** (https://neon.tech): Excellent free tier, serverless Postgres.
86
+ 2. **Railway** (https://railway.app): Easy Postgres provisioning.
87
+ 3. **Supabase** (https://supabase.com): Create a project and grab the Postgres connection string from Settings > Database. (You do not need the Supabase JS SDK, just the pure Postgres string).
88
+
89
+ *Note: Your `DATABASE_URL` must start with `postgresql+asyncpg://` for our SQLAlchemy setup.*
90
+
91
+ ### Alembic Migrations
92
+ When your HuggingFace space boots up, the `start.sh` script automatically runs `alembic upgrade head`. This ensures your database schema is perfectly in sync with your code!
93
+
94
+ ---
95
+
96
+ ## Post-Deployment Validation
97
+
98
+ ### 1. Verify Deployment
99
+ - Access your Space: `https://datavision-ai-datavision.hf.space`
100
+ - Check HuggingFace logs for any runtime errors.
101
+ - Create an account and log in.
102
+
103
+ ### 2. Check Persistent Storage
104
+ HuggingFace Spaces comes with 50GB of persistent storage mounted at `/app/storage` (where we configured it in the Dockerfile).
105
+ - Upload a file and verify it persists after the space reboots.
106
+ - Check that trained AutoML models and vector FAISS indexes are saved properly.
107
+
108
+ ---
109
+
110
+ ## Troubleshooting
111
+
112
+ ### Issue: GitHub Action Fails on Deploy
113
+ **Fix:** Ensure you added the `HF_TOKEN` exactly as specified in the GitHub Secrets. Check the action logs for specific Git errors.
114
+
115
+ ### Issue: "Application Startup Failed" in HF Logs
116
+ **Fix:** Likely a database connection issue or missing `GROQ_API_KEY`. Verify all secrets in HF Settings and ensure your `DATABASE_URL` is accessible from the internet.
117
+
118
+ ### Issue: "Database schema out of date"
119
+ **Fix:** Restart the space. The `start.sh` script will run `alembic upgrade head` again.
120
+
121
+ ---
122
+
123
+ See `MLOPS.md` for complete architecture and lifecycle details.
MLOPS.md ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DataVision AI - Complete MLOps Architecture & Lifecycle
2
+
3
+ This document describes the complete lifecycle of DataVision AI, from source code to model training, deployment, and monitoring on HuggingFace Spaces.
4
+
5
+ ## 1. System Architecture
6
+
7
+ The application is deployed as a single monolithic Docker container on HuggingFace Spaces for simplicity and cost-effectiveness.
8
+
9
+ ```mermaid
10
+ graph TD
11
+ A[User] -->|HTTPS| B(HuggingFace Space :7860)
12
+ B -->|Serves Static Files| C[React Frontend]
13
+ B -->|API Routes| D[FastAPI Backend]
14
+ D <-->|PostgreSQL| E[(External Hosted DB)]
15
+ D <-->|Persistent Storage| F[(HF Space /app/storage)]
16
+ D <-->|LLM Queries| G(Groq / OpenAI API)
17
+ ```
18
+
19
+ ## 2. CI/CD Pipeline
20
+
21
+ We utilize GitHub Actions to provide a zero-touch deployment experience.
22
+
23
+ 1. **Local Development**: Developer works on code locally and tests using `docker-compose up`.
24
+ 2. **Push to Main**: `git push origin main` triggers the GitHub Actions pipeline.
25
+ 3. **CI Checks (`ci.yml`)**:
26
+ - Runs `ruff` for Python linting and syntax checking.
27
+ - Runs `npm run build` to ensure the React frontend compiles cleanly.
28
+ 4. **Deployment (`deploy.yml`)**:
29
+ - Pushes the source code (including the freshly built `frontend/dist`) to the HuggingFace remote using the `HF_TOKEN`.
30
+ 5. **HuggingFace Build**:
31
+ - HF detects the push, spins up a Build server, and executes the `Dockerfile`.
32
+ - Python dependencies are installed.
33
+ 6. **Container Startup (`start.sh`)**:
34
+ - Space awakens.
35
+ - `alembic upgrade head` executes to apply any new database schema migrations to the external PostgreSQL database.
36
+ - `uvicorn` starts the FastAPI server on port 7860.
37
+
38
+ ## 3. Storage & Persistence Strategy
39
+
40
+ Because Docker containers on HuggingFace are ephemeral (they sleep after inactivity and get completely rebuilt on new commits), persistence must be explicitly managed.
41
+
42
+ ### Database (Structured Data)
43
+ - **What**: Users, channels, chat threads, report metadata, model metadata.
44
+ - **Where**: External PostgreSQL database (e.g., Neon).
45
+ - **Why**: Ephemeral SQLite would wipe user accounts every time the space sleeps.
46
+
47
+ ### File Storage (Unstructured/Large Data)
48
+ - **What**: Uploaded CSV/Excel files, trained ML models (.pkl, .joblib), generated PDF/HTML reports, FAISS vector indexes.
49
+ - **Where**: `/app/storage` inside the container.
50
+ - **Why**: HuggingFace Spaces provides 50GB of persistent storage mounted at this directory. Our `Dockerfile` explicitly sets permissions on this folder to ensure the `appuser` can write to it.
51
+
52
+ ## 4. Machine Learning Lifecycle
53
+
54
+ DataVision features an autonomous AutoML pipeline.
55
+
56
+ 1. **Ingestion**: User uploads a CSV. It is parsed using pandas and saved to `/app/storage/uploads`.
57
+ 2. **Preprocessing**: The Agentic system analyzes the data, imputes missing values, and encodes categoricals.
58
+ 3. **Training**: LightGBM/CatBoost/XGBoost models are trained in memory.
59
+ 4. **Persistence**: The best performing model pipeline is serialized using `joblib` and saved to `/app/storage/models/{model_id}.pkl`. Model metadata is saved to PostgreSQL.
60
+ 5. **Deployment**: When a user queries the model, the backend uses `model_deployer.py` to deserialize the model into memory.
61
+ 6. **Inference**: User provides new data; the in-memory engine predicts the outcome.
62
+ 7. **Retirement**: The user can delete the model via the UI, which soft-deletes it in Postgres and removes the `.pkl` file from persistent storage.
63
+
64
+ ## 5. Security Practices
65
+
66
+ - **Non-root Execution**: The `Dockerfile` creates a dedicated `appuser`. The application does not run as root.
67
+ - **Secrets Management**: No API keys are in the codebase. All keys (`GROQ_API_KEY`, `DATABASE_URL`, `JWT_SECRET`) are securely injected via HuggingFace Space Secrets at runtime.
68
+ - **Rate Limiting**: The `core/rate_limiter.py` module protects expensive endpoints (like LLM generation and model training) from abuse.
69
+ - **Data Isolation**: Files are stored in user-specific directories (`/app/storage/users/{user_id}/`).
70
+
71
+ ## 6. Rollback Procedure
72
+
73
+ If a deployment fails or introduces a critical bug:
74
+ 1. Revert the commit locally: `git revert HEAD`
75
+ 2. Push to main: `git push origin main`
76
+ 3. The CI/CD pipeline will automatically build and deploy the reverted, stable code.
77
+
78
+ If database migrations caused the issue, you may need to manually downgrade the database using Alembic before reverting the code, or restore the database from a backup via your Postgres provider (e.g. Neon's point-in-time recovery).
PRD.md ADDED
@@ -0,0 +1,838 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 📊 DataVision AI - Product Requirements Document (PRD)
2
+
3
+ **Document Version:** 1.0
4
+ **Last Updated:** January 2025
5
+ **Product Name:** DataVision AI
6
+ **Tagline:** *"Your data doesn't need dashboards. It needs intelligence."*
7
+
8
+ ---
9
+
10
+ ## 📋 Table of Contents
11
+
12
+ 1. [Executive Summary](#1-executive-summary)
13
+ 2. [Product Vision & Mission](#2-product-vision--mission)
14
+ 3. [Target Audience & User Personas](#3-target-audience--user-personas)
15
+ 4. [Core Features & Capabilities](#4-core-features--capabilities)
16
+ 5. [Technical Architecture](#5-technical-architecture)
17
+ 6. [API Reference](#6-api-reference)
18
+ 7. [AI/ML Capabilities](#7-aiml-capabilities)
19
+ 8. [User Journeys & Flows](#8-user-journeys--flows)
20
+ 9. [Security & Compliance](#9-security--compliance)
21
+ 10. [Success Metrics & KPIs](#10-success-metrics--kpis)
22
+ 11. [Roadmap & Future Enhancements](#11-roadmap--future-enhancements)
23
+
24
+ ---
25
+
26
+ ## 1. Executive Summary
27
+
28
+ ### 1.1 Product Overview
29
+
30
+ **DataVision AI** is an autonomous AI-powered business intelligence platform that transforms raw data into actionable executive insights. It combines state-of-the-art **AutoML** (Automated Machine Learning), **natural language AI chat**, and **intelligent reporting** into a unified "Business Analyst in a Box" experience.
31
+
32
+ ### 1.2 Problem Statement
33
+
34
+ Traditional business intelligence tools require:
35
+ - Technical expertise to build dashboards and reports
36
+ - Data science knowledge for predictive modeling
37
+ - SQL skills for data querying
38
+ - Weeks of implementation time
39
+
40
+ **Result:** 73% of enterprise data goes unanalyzed, and business users remain dependent on technical teams.
41
+
42
+ ### 1.3 Solution
43
+
44
+ DataVision AI democratizes data analysis by providing:
45
+
46
+ | Capability | Traditional BI | DataVision AI |
47
+ |------------|----------------|---------------|
48
+ | Dashboard Creation | Manual, hours | AI-generated, seconds |
49
+ | Predictive Models | Data science team | Zero-code AutoML |
50
+ | Data Queries | SQL required | Natural language |
51
+ | Reports | Manual creation | AI-automated |
52
+ | Time to Insight | Days/weeks | Minutes |
53
+
54
+ ### 1.4 Key Differentiators
55
+
56
+ 1. **Zero-Code AutoML** - Train production ML models without writing code
57
+ 2. **Autonomous Dashboards** - AI designs the entire dashboard automatically
58
+ 3. **7 AI Modes** - Different reasoning approaches for different analytical needs
59
+ 4. **Real-time Prediction Playground** - Interactive predictions with instant results
60
+ 5. **Multi-Agent System** - 40+ specialized AI agents for comprehensive analysis
61
+ 6. **Advanced RAG System** - 3-tier retrieval for accurate, context-aware answers
62
+ 7. **Enterprise-Ready** - Security, audit trails, scheduled reports
63
+
64
+ ---
65
+
66
+ ## 2. Product Vision & Mission
67
+
68
+ ### 2.1 Vision Statement
69
+
70
+ > *"To make every business user a data-powered decision maker by removing technical barriers between humans and their data."*
71
+
72
+ ### 2.2 Mission
73
+
74
+ Empower organizations to:
75
+ - **Analyze** data instantly through natural language
76
+ - **Predict** future outcomes with production-ready ML models
77
+ - **Automate** reporting and dashboard generation
78
+ - **Democratize** data science across all business functions
79
+
80
+ ### 2.3 Core Value Propositions
81
+
82
+ | For Business Users | For Data Teams | For Executives |
83
+ |-------------------|----------------|----------------|
84
+ | Ask questions in plain English | Focus on complex problems | Instant executive summaries |
85
+ | Get instant visualizations | Reduce ad-hoc requests by 80% | AI-designed KPI dashboards |
86
+ | Make predictions without code | Production-ready model exports | Scheduled automated reports |
87
+ | Self-service analytics | Maintain data governance | Real-time business insights |
88
+
89
+ ---
90
+
91
+ ## 3. Target Audience & User Personas
92
+
93
+ ### 3.1 Primary User Personas
94
+
95
+ #### 👤 Persona 1: Business Analyst (Sarah)
96
+
97
+ | Attribute | Details |
98
+ |-----------|---------|
99
+ | **Role** | Senior Business Analyst at mid-size company |
100
+ | **Technical Level** | Low (knows Excel, no SQL/Python) |
101
+ | **Pain Points** | Dependent on data team, slow report turnaround |
102
+ | **Goals** | Self-service analytics, faster insights |
103
+ | **Primary Features** | AI Chat, Reports, Dashboard |
104
+ | **Success Metric** | Reduce time-to-insight from days to minutes |
105
+
106
+ #### 👤 Persona 2: Data Scientist (Alex)
107
+
108
+ | Attribute | Details |
109
+ |-----------|---------|
110
+ | **Role** | Data Scientist at tech company |
111
+ | **Technical Level** | High (Python, ML frameworks) |
112
+ | **Pain Points** | Repetitive modeling tasks, stakeholder requests |
113
+ | **Goals** | Rapid prototyping, model comparison |
114
+ | **Primary Features** | AutoML, ML Predictions, Feature Engineering |
115
+ | **Success Metric** | Reduce model development time by 10x |
116
+
117
+ #### 👤 Persona 3: Executive (Michael)
118
+
119
+ | Attribute | Details |
120
+ |-----------|---------|
121
+ | **Role** | VP of Operations at enterprise |
122
+ | **Technical Level** | None |
123
+ | **Pain Points** | Outdated reports, no real-time visibility |
124
+ | **Goals** | Real-time KPIs, predictive insights |
125
+ | **Primary Features** | Executive Reports, Autonomous Dashboard |
126
+ | **Success Metric** | Data-driven decision making |
127
+
128
+ #### 👤 Persona 4: Product Manager (Priya)
129
+
130
+ | Attribute | Details |
131
+ |-----------|---------|
132
+ | **Role** | Product Manager at SaaS startup |
133
+ | **Technical Level** | Medium (can read data, limited SQL) |
134
+ | **Pain Points** | Understanding user trends, forecasting |
135
+ | **Goals** | User segmentation, feature impact analysis |
136
+ | **Primary Features** | Forecasting, Anomaly Detection, Chat |
137
+ | **Success Metric** | Identify growth opportunities |
138
+
139
+ ### 3.2 Key Use Cases
140
+
141
+ | Use Case | Description | User Persona |
142
+ |----------|-------------|--------------|
143
+ | **Quick Data Analysis** | Drop CSV → Get instant insights | Business Analyst |
144
+ | **Predictive Modeling** | Train ML models without code | Data Scientist |
145
+ | **Automated Reporting** | Generate executive reports on demand | Executive |
146
+ | **Conversational Analytics** | Ask natural language questions | All |
147
+ | **Revenue Forecasting** | Predict future metrics | Product Manager |
148
+ | **Anomaly Detection** | Find outliers and unusual patterns | Operations |
149
+ | **Customer Segmentation** | Auto-cluster users/customers | Marketing |
150
+ | **Root Cause Analysis** | Understand "why" metrics changed | Analyst |
151
+
152
+ ---
153
+
154
+ ## 4. Core Features & Capabilities
155
+
156
+ ### 4.1 Data Management (DataHub)
157
+
158
+ The central hub for all data operations.
159
+
160
+ | Feature | Description | Status |
161
+ |---------|-------------|--------|
162
+ | **Multi-format Upload** | CSV, Excel (.xlsx, .xls), JSON, PDF, DOCX, TXT, Images | ✅ Live |
163
+ | **Auto Data Quality Fix** | One-click data cleaning (missing values, outliers, duplicates) | ✅ Live |
164
+ | **Smart Column Detection** | Automatic target column detection for ML | ✅ Live |
165
+ | **Google Sheets Import** | Direct import from Google Sheets URLs | ✅ Live |
166
+ | **Multi-file Training** | Combine multiple files for unified training | ✅ Live |
167
+ | **Upload Cancellation** | Cancel in-progress uploads with cleanup | ✅ Live |
168
+ | **File Size Limit** | 100MB per file | ✅ Live |
169
+ | **Data Profiling** | Automatic statistical summaries | ✅ Live |
170
+ | **Currency Detection** | Auto-detect INR, USD, EUR, GBP, etc. | ✅ Live |
171
+
172
+ ### 4.2 AutoML Training System
173
+
174
+ Production-ready machine learning without code.
175
+
176
+ #### Training Modes
177
+
178
+ | Mode | Algorithms | Duration | Use Case |
179
+ |------|------------|----------|----------|
180
+ | **Fast Mode** | 7 algorithms | 30-60 seconds | Quick prototyping |
181
+ | **Ultra Mode** | 20+ algorithms | 2-10 minutes | Production models |
182
+
183
+ #### Supported Algorithms
184
+
185
+ **Classification (15+ algorithms):**
186
+ - XGBoost Classifier
187
+ - LightGBM Classifier
188
+ - CatBoost Classifier
189
+ - Random Forest Classifier
190
+ - Extra Trees Classifier
191
+ - Gradient Boosting Classifier
192
+ - AdaBoost Classifier
193
+ - Logistic Regression
194
+ - Support Vector Machine (SVM)
195
+ - K-Nearest Neighbors (KNN)
196
+ - Neural Network (MLP)
197
+ - Naive Bayes (Gaussian, Multinomial)
198
+ - Decision Tree Classifier
199
+ - Bagging Classifier
200
+
201
+ **Regression (15+ algorithms):**
202
+ - XGBoost Regressor
203
+ - LightGBM Regressor
204
+ - CatBoost Regressor
205
+ - Random Forest Regressor
206
+ - Extra Trees Regressor
207
+ - Gradient Boosting Regressor
208
+ - Ridge Regression
209
+ - Lasso Regression
210
+ - ElasticNet
211
+ - Bayesian Ridge
212
+ - Huber Regressor
213
+ - Poisson Regressor
214
+ - Quantile Regressor
215
+ - Neural Network (MLP)
216
+
217
+ **Clustering:**
218
+ - K-Means
219
+ - DBSCAN
220
+ - Agglomerative Clustering
221
+ - Spectral Clustering
222
+ - Gaussian Mixture Model
223
+ - MeanShift
224
+
225
+ #### AutoML Features
226
+
227
+ | Feature | Description |
228
+ |---------|-------------|
229
+ | **Auto Task Detection** | Classification, Regression, Clustering auto-detection |
230
+ | **Feature Engineering** | 50+ synthetic features generation |
231
+ | **Hyperparameter Optimization** | Bayesian optimization with Optuna |
232
+ | **Ensemble Methods** | Stacking, Voting, Blending |
233
+ | **GPU Acceleration** | Auto GPU/CPU detection (CUDA, ROCm, Metal) |
234
+ | **Class Imbalance Handling** | SMOTE, class weights |
235
+ | **NLP Pipeline** | TF-IDF, Sentiment, Text Stats for text columns |
236
+ | **Cross-Validation** | Stratified K-Fold for robust evaluation |
237
+ | **Training Stop** | Cancel training mid-process |
238
+
239
+ ### 4.3 ML Predictions & Analysis
240
+
241
+ Interactive prediction and model analysis tools.
242
+
243
+ | Feature | Description |
244
+ |---------|-------------|
245
+ | **Prediction Playground** | Interactive sliders for real-time predictions |
246
+ | **Batch Predictions** | Predict on entire datasets |
247
+ | **Model Explainability** | SHAP values and feature contributions |
248
+ | **Confusion Matrix** | Classification performance visualization |
249
+ | **ROC/PR Curves** | Model performance curves |
250
+ | **Feature Importance** | Visual ranking of predictive features |
251
+ | **What-If Analysis** | Scenario planning with predictions |
252
+ | **Model History** | Version control with rollback |
253
+ | **Model Persistence** | Save, load, delete trained models |
254
+
255
+ ### 4.4 Autonomous Dashboard
256
+
257
+ AI-designed dashboards that build themselves.
258
+
259
+ | Feature | Description |
260
+ |---------|-------------|
261
+ | **AI-Designed Layout** | LLM decides KPIs, charts, colors automatically |
262
+ | **15+ Chart Types** | Bar, Line, Pie, Scatter, Heatmap, Sunburst, Treemap, etc. |
263
+ | **Dynamic KPIs** | Auto-calculated business metrics |
264
+ | **Domain Detection** | Auto-detect data domain (sales, finance, HR, etc.) |
265
+ | **Real-time Updates** | Dashboard refreshes when data changes |
266
+ | **Multiple Views** | Grid and List display modes |
267
+ | **Chart Interactions** | Hover, zoom, expand capabilities |
268
+
269
+ ### 4.5 AI Analyst Chat
270
+
271
+ Conversational analytics powered by LLMs.
272
+
273
+ | Feature | Description |
274
+ |---------|-------------|
275
+ | **Natural Language Queries** | Ask anything about your data |
276
+ | **7 AI Modes** | Analyst, DeepThink, Vision, Predict, Agent, RAG, MCP |
277
+ | **ChatGPT-style Typing** | Word-by-word response animation |
278
+ | **Interactive Charts** | Plotly charts generated from queries |
279
+ | **Voice Input** | Microphone support for queries |
280
+ | **File Attachments** | Attach files directly in chat |
281
+ | **Conversation History** | Per-user conversation memory |
282
+ | **Smart Suggestions** | Follow-up question recommendations |
283
+ | **Confidence Scoring** | Answer reliability indicators |
284
+
285
+ #### AI Chat Modes
286
+
287
+ | Mode | Icon | Description | Best For |
288
+ |------|------|-------------|----------|
289
+ | **Analyst** | 📊 | Business analyst-style insights | General analysis |
290
+ | **DeepThink** | 🧠 | Multi-step chain-of-thought reasoning | Complex questions |
291
+ | **Vision** | 👁️ | Data visualization focus | Charts and graphs |
292
+ | **Predict** | 🔮 | ML predictions and forecasts | Future predictions |
293
+ | **Agent** | 🤖 | Autonomous multi-agent orchestration | Complex tasks |
294
+ | **RAG** | 📚 | Retrieval-augmented generation | Document Q&A |
295
+ | **MCP** | 🔗 | Model Context Protocol | Tool integration |
296
+
297
+ ### 4.6 Reports Generation
298
+
299
+ Automated, AI-powered business reports.
300
+
301
+ | Report Type | Description | Output |
302
+ |-------------|-------------|--------|
303
+ | **Metrics Analysis** | Numeric data trends and patterns | Text + Charts |
304
+ | **Data Breakdown** | Category distributions and segments | Text + Charts |
305
+ | **Executive Summary** | High-level insights for leadership | PDF/Text |
306
+ | **Predictive Report** | ML forecasts (requires trained model) | Text + Charts |
307
+ | **Anomaly Report** | Outlier and unusual pattern detection | Text + Charts |
308
+
309
+ **Export Options:**
310
+ - PDF Export (branded, formatted)
311
+ - Text Export (downloadable)
312
+ - Multi-currency formatting (INR, USD, EUR, GBP)
313
+
314
+ ### 4.7 Advanced RAG System
315
+
316
+ Three-tier retrieval-augmented generation for accurate answers.
317
+
318
+ | Tier | Features |
319
+ |------|----------|
320
+ | **Tier 1 (Standard)** | Query decomposition, MMR reranking, answer evaluation |
321
+ | **Tier 2 (Advanced)** | HyDE, Corrective RAG, Self-Reflection |
322
+ | **Tier 3 (Agentic)** | Tool-using agents, Multi-source retrieval, RRF fusion |
323
+
324
+ **RAG Components:**
325
+ - FAISS Vector Store for semantic search
326
+ - Knowledge Graph for relationship mapping
327
+ - Semantic query caching
328
+ - Context window optimization
329
+
330
+ ---
331
+
332
+ ## 5. Technical Architecture
333
+
334
+ ### 5.1 System Architecture
335
+
336
+ ```
337
+ ┌─────────────────────────────────────────────────────────────────────┐
338
+ │ FRONTEND (React 18) │
339
+ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
340
+ │ │ DataHub │ │ AutoML │ │ Dashboard│ │ AI Chat │ │ Reports │ │
341
+ │ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
342
+ └───────┼────────────┼────────────┼────────────┼────────────┼────────┘
343
+ │ │ │ │ │
344
+ └────────────┴────────────┴────────────┴────────────┘
345
+ │
346
+ ┌─────────────▼─────────────┐
347
+ │ API GATEWAY (FastAPI) │
348
+ │ Rate Limiting | Auth │
349
+ └─────────────┬─────────────┘
350
+ │
351
+ ┌───────────────┬─────────┴─────────┬───────────────┐
352
+ │ │ │ │
353
+ ┌───────▼───────┐ ┌─────▼─────┐ ┌───────────▼───────────┐ ┌─▼──────────┐
354
+ │ Data Service │ │ ML Engine │ │ AI Agent System │ │ RAG Core │
355
+ │ - Ingestion │ │ - AutoML │ │ - 40+ Specialized │ │ - FAISS │
356
+ │ - Quality │ │ - Train │ │ Agents │ │ - Graph │
357
+ │ - Transform │ │ - Predict │ │ - Orchestrator │ │ - Cache │
358
+ └───────┬───────┘ └─────┬─────┘ └───────────┬───────────┘ └────────────┘
359
+ │ │ │
360
+ └───────────────┴───────────────────┘
361
+ │
362
+ ┌───────────────┼───────────────┐
363
+ │ │ │
364
+ ┌───────▼───────┐ ┌─────▼─────┐ ┌───────▼───────┐
365
+ │ File Store │ │ Models │ │ Supabase │
366
+ │ /storage/data │ │ /models │ │ PostgreSQL │
367
+ └───────────────┘ └───────────┘ └───────────────┘
368
+ ```
369
+
370
+ ### 5.2 Technology Stack
371
+
372
+ | Layer | Technologies |
373
+ |-------|-------------|
374
+ | **Frontend** | React 18, Vite, TypeScript, Tailwind CSS, Framer Motion |
375
+ | **UI Components** | Radix UI, Headless UI, Lucide Icons |
376
+ | **Charts** | Plotly.js, Recharts, ApexCharts |
377
+ | **Backend** | FastAPI, Python 3.11+, Uvicorn, Pydantic |
378
+ | **ML Framework** | Scikit-learn, XGBoost, LightGBM, CatBoost, Optuna |
379
+ | **AI/LLM** | OpenAI GPT-4, Groq, Claude, OpenRouter |
380
+ | **Vector DB** | FAISS |
381
+ | **Database** | Supabase (PostgreSQL) |
382
+ | **Authentication** | Supabase Auth (JWT) |
383
+ | **Email** | Resend API |
384
+ | **Deployment** | Docker, HuggingFace Spaces |
385
+ | **File Processing** | Pandas, PyMuPDF, python-docx, Pillow |
386
+
387
+ ### 5.3 Backend Structure
388
+
389
+ ```
390
+ backend/
391
+ ├── main.py # FastAPI application entry point
392
+ ├── api/
393
+ │ └── v1/
394
+ │ └── endpoints/ # All API endpoint handlers
395
+ ├── agents/ # 40+ AI agents
396
+ │ ├── orchestrator.py # Multi-agent coordinator
397
+ │ ├── universal_agent.py # Query router
398
+ │ ├── forecast_agent.py # Forecasting
399
+ │ ├── memory_agent.py # Conversation memory
400
+ │ └── ...
401
+ ├── core/ # Business logic engines
402
+ │ ├── autonomous_data_ops.py
403
+ │ ├── export_engine.py
404
+ │ ├── data_grounding.py
405
+ │ └── ...
406
+ ├── ml/ # ML pipeline components
407
+ │ ├── automl_engine.py
408
+ │ ├── feature_engineer.py
409
+ │ └── ...
410
+ ├── graph/ # Knowledge graph
411
+ ├── vector/ # FAISS vector store
412
+ ├── ingestion/ # Data ingestion pipeline
413
+ ├── services/ # External services (email, etc.)
414
+ ├── scheduler/ # Scheduled reports
415
+ └── utils/ # Utility functions
416
+ ```
417
+
418
+ ### 5.4 Frontend Structure
419
+
420
+ ```
421
+ frontend/
422
+ ├── src/
423
+ │ ├── pages/ # 18 main pages
424
+ │ │ ├── Landing.tsx # Landing page
425
+ │ │ ├── DataHub.tsx # Data management
426
+ │ │ ├── AutoML.tsx # ML training
427
+ │ │ ├── MLPredictions.tsx
428
+ │ │ ├── AutonomousDashboard.tsx
429
+ │ │ ├── AnalystChat.tsx
430
+ │ │ ├── Reports.tsx
431
+ │ │ └── ...
432
+ │ ├── components/ # Reusable UI components
433
+ │ ├── services/ # API client
434
+ │ ├── store/ # Zustand state management
435
+ │ ├── contexts/ # React contexts
436
+ │ └── lib/ # Utilities
437
+ └── public/ # Static assets
438
+ ```
439
+
440
+ ---
441
+
442
+ ## 6. API Reference
443
+
444
+ ### 6.1 Authentication APIs
445
+
446
+ | Endpoint | Method | Description |
447
+ |----------|--------|-------------|
448
+ | `/api/v1/email-prefs/auth/request-password-reset` | POST | Request password reset email |
449
+ | `/api/v1/email-prefs/auth/update-password-with-token` | POST | Update password with token |
450
+
451
+ ### 6.2 Data Management APIs
452
+
453
+ | Endpoint | Method | Description |
454
+ |----------|--------|-------------|
455
+ | `/api/v1/upload/` | POST | Upload files |
456
+ | `/api/v1/files/` | GET | List user files |
457
+ | `/api/v1/files/{filename}` | DELETE | Delete file |
458
+ | `/api/v1/files/cancel-upload` | POST | Cancel upload |
459
+ | `/api/v2/data/quality-check` | POST | Check data quality |
460
+ | `/api/v2/data/auto-fix` | POST | Auto-fix data issues |
461
+ | `/api/v1/upload/google-sheets` | POST | Import from Google Sheets |
462
+
463
+ ### 6.3 Analytics APIs
464
+
465
+ | Endpoint | Method | Description |
466
+ |----------|--------|-------------|
467
+ | `/api/v1/analytics/overview` | GET | Get analytics overview |
468
+ | `/api/v1/unified-analytics/` | POST | Unified analytics engine |
469
+ | `/api/v1/universal/` | POST | Universal AI agent |
470
+ | `/api/v1/chat/` | POST | AI chat endpoint |
471
+ | `/api/v1/autonomous/dashboard` | POST | Generate dashboard |
472
+ | `/api/v1/autonomous/auto-analyze` | POST | Auto-analyze file |
473
+
474
+ ### 6.4 AutoML APIs (v2)
475
+
476
+ | Endpoint | Method | Description |
477
+ |----------|--------|-------------|
478
+ | `/api/v2/automl/train` | POST | Fast mode training |
479
+ | `/api/v2/automl/ultra-train` | POST | Ultra mode training |
480
+ | `/api/v2/automl/predict` | POST | Make predictions |
481
+ | `/api/v2/automl/batch-predict` | POST | Batch predictions |
482
+ | `/api/v2/automl/charts` | GET | Get ML charts |
483
+ | `/api/v2/automl/stop` | POST | Stop training |
484
+ | `/api/v2/automl/models` | GET | List models |
485
+ | `/api/v2/automl/models/{id}` | DELETE | Delete model |
486
+ | `/api/v2/automl/rollback` | POST | Rollback model |
487
+
488
+ ### 6.5 Prediction Playground APIs
489
+
490
+ | Endpoint | Method | Description |
491
+ |----------|--------|-------------|
492
+ | `/api/v2/playground/config` | GET | Get playground config |
493
+ | `/api/v2/playground/predict` | POST | Playground prediction |
494
+ | `/api/v2/playground/shap` | POST | SHAP explanation |
495
+
496
+ ### 6.6 Reports APIs
497
+
498
+ | Endpoint | Method | Description |
499
+ |----------|--------|-------------|
500
+ | `/api/v1/reports/generate` | POST | Generate report |
501
+ | `/api/v1/reports/export/pdf` | POST | Export PDF |
502
+ | `/api/v1/reports/scheduled` | POST | Schedule report |
503
+
504
+ ### 6.7 Storage APIs
505
+
506
+ | Endpoint | Method | Description |
507
+ |----------|--------|-------------|
508
+ | `/api/v1/autonomous/models/{user_id}` | GET | List user models |
509
+ | `/api/v1/autonomous/models/{user_id}` | DELETE | Delete all models |
510
+
511
+ ---
512
+
513
+ ## 7. AI/ML Capabilities
514
+
515
+ ### 7.1 AI Agent System
516
+
517
+ DataVision AI employs a sophisticated multi-agent architecture with 40+ specialized agents.
518
+
519
+ #### Core Agents
520
+
521
+ | Agent | File | Purpose |
522
+ |-------|------|---------|
523
+ | **Universal Agent** | `universal_agent.py` | NLU query routing and intent detection |
524
+ | **Orchestrator** | `orchestrator.py` | Multi-agent coordination |
525
+ | **Query Classifier** | `query_classifier.py` | Intent classification |
526
+ | **Query Dispatcher** | `query_dispatcher.py` | Route to appropriate handler |
527
+ | **Response Enhancer** | `response_enhancer.py` | Answer quality improvement |
528
+ | **Confidence Scorer** | `confidence_scorer.py` | Answer reliability scoring |
529
+ | **Memory Agent** | `memory_agent.py` | Conversation context management |
530
+
531
+ #### Analytics Agents
532
+
533
+ | Agent | File | Purpose |
534
+ |-------|------|---------|
535
+ | **Forecast Agent** | `forecast_agent.py` | Revenue and metric forecasting |
536
+ | **Deep Research Agent** | `core/deep_research_agent.py` | Multi-step research |
537
+ | **Data Quality Agent** | `data_quality.py` | Data validation and cleaning |
538
+ | **Visualization Agent** | `visualization.py` | Chart generation |
539
+
540
+ #### ML Pipeline Agents
541
+
542
+ | Agent | File | Purpose |
543
+ |-------|------|---------|
544
+ | **Feature Engineer** | `feature_engineer.py` | Feature creation |
545
+ | **Model Strategy Agent** | `model_strategy.py` | Algorithm selection |
546
+ | **Hyperparameter Agent** | `hyperparam.py` | Optimization |
547
+ | **Training Validator** | `training_validator.py` | Training validation |
548
+ | **Evaluation Agent** | `evaluation.py` | Model evaluation |
549
+ | **Deployment Agent** | `deployment.py` | Production preparation |
550
+
551
+ #### Chart Agents
552
+
553
+ | Agent | File | Purpose |
554
+ |-------|------|---------|
555
+ | **Chart Gatekeeper** | `chart_gatekeeper.py` | Chart validation |
556
+ | **Smart Chart** | `smart_chart.py` | LLM-driven chart generation |
557
+ | **Premium Charts** | `premium_charts.py` | Advanced visualizations |
558
+ | **Autonomous Charts** | `autonomous_charts.py` | Auto dashboard charts |
559
+
560
+ ### 7.2 LLM Integration
561
+
562
+ | Provider | Models | Use Case |
563
+ |----------|--------|----------|
564
+ | **OpenAI** | GPT-4, GPT-4-turbo, GPT-3.5-turbo | Primary inference |
565
+ | **Groq** | Llama 3, Mixtral | Fast inference |
566
+ | **Claude** | Claude 3, Claude 3.5 | Complex reasoning |
567
+ | **OpenRouter** | Multi-model routing | Fallback and routing |
568
+
569
+ ### 7.3 ML Model Performance Targets
570
+
571
+ | Task | Metric | Target | Notes |
572
+ |------|--------|--------|-------|
573
+ | Classification | Accuracy | >85% | With hyperparameter tuning |
574
+ | Classification | F1-Score | >0.80 | For imbalanced datasets |
575
+ | Regression | R² | >0.75 | For well-structured data |
576
+ | Regression | RMSE | <15% of range | Normalized |
577
+ | Clustering | Silhouette | >0.5 | For clear clusters |
578
+
579
+ ---
580
+
581
+ ## 8. User Journeys & Flows
582
+
583
+ ### 8.1 New User Onboarding Flow
584
+
585
+ ```
586
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
587
+ │ Landing │───▶│ Signup │───▶│ Confirm │
588
+ │ Page │ │ Page │ │ Email │
589
+ └──────────────┘ └──────────────┘ └──────────────┘
590
+ │
591
+ ▼
592
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
593
+ │ DataHub │◀───│ Tutorial │◀───│ Login │
594
+ │ (Upload CSV) │ │ (Optional) │ │ Page │
595
+ └──────────────┘ └──────────────┘ └──────────────┘
596
+ ```
597
+
598
+ ### 8.2 Data Analysis Flow
599
+
600
+ ```
601
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
602
+ │ Upload │───▶│ Quality │───▶│ Auto-Fix │
603
+ │ File │ │ Check │ │ (Optional) │
604
+ └──────────────┘ └──────────────┘ └──────────────┘
605
+ │
606
+ ┌──────────────────────────────────────┤
607
+ │ │
608
+ ▼ ▼
609
+ ┌──────────────┐ ┌──────────────┐
610
+ │ AI Chat │ │ AutoML │
611
+ │ Analysis │ │ Training │
612
+ └──────────────┘ └──────────────┘
613
+ │ │
614
+ ▼ ▼
615
+ ┌──────────────┐ ┌──────────────┐
616
+ │ Reports │ │ Predictions │
617
+ │ Generation │ │ Playground │
618
+ └──────────────┘ └──────────────┘
619
+ ```
620
+
621
+ ### 8.3 AutoML Training Flow
622
+
623
+ ```
624
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
625
+ │ Select │───▶│ Choose │───▶│ Configure │
626
+ │ File │ │ Target │ │ Mode │
627
+ └──────────────┘ └──────────────┘ └──────────────┘
628
+ │
629
+ ▼
630
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
631
+ │ Model │◀───│ Training │◀───│ Start │
632
+ │ Results │ │ Progress │ │ Training │
633
+ └──────────────┘ └──────────────┘ └──────────────┘
634
+ │
635
+ ├───────────────┬───────────────┐
636
+ ▼ ▼ ▼
637
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
638
+ │ Playground │ │ Charts │ │ Predictions │
639
+ │ Prediction │ │ (ROC/CM) │ │ Batch │
640
+ └──────────────┘ └──────────────┘ └──────────────┘
641
+ ```
642
+
643
+ ### 8.4 Dashboard Generation Flow
644
+
645
+ ```
646
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
647
+ │ Select │───▶│ AI │───▶│ Display │
648
+ │ File │ │ Analyzes │ │ Dashboard │
649
+ └──────────────┘ └──────────────┘ └──────────────┘
650
+ │
651
+ │ Auto-generates:
652
+ ├── KPI Cards
653
+ ├── Chart Types
654
+ ├── Color Schemes
655
+ └── Layout
656
+ ```
657
+
658
+ ---
659
+
660
+ ## 9. Security & Compliance
661
+
662
+ ### 9.1 Authentication & Authorization
663
+
664
+ | Feature | Implementation |
665
+ |---------|---------------|
666
+ | **User Authentication** | Supabase Auth with JWT |
667
+ | **Password Policy** | Minimum 8 characters |
668
+ | **Password Reset** | Custom branded email flow |
669
+ | **Session Management** | JWT with refresh tokens |
670
+ | **Rate Limiting** | Per-user request limits |
671
+
672
+ ### 9.2 Data Security
673
+
674
+ | Feature | Implementation |
675
+ |---------|---------------|
676
+ | **Data Isolation** | Per-user file and model storage |
677
+ | **File Sanitization** | Filename sanitization, path traversal prevention |
678
+ | **Input Validation** | Pydantic models for all inputs |
679
+ | **Secure Headers** | XSS, CSRF protection |
680
+ | **AI Security** | Prompt injection detection |
681
+
682
+ ### 9.3 Data Privacy
683
+
684
+ | Aspect | Approach |
685
+ |--------|----------|
686
+ | **Data Storage** | User files stored in isolated directories |
687
+ | **Data Retention** | User-controlled deletion |
688
+ | **Data Processing** | In-memory processing, no permanent storage of analysis |
689
+ | **Model Storage** | User-isolated model directories |
690
+
691
+ ### 9.4 Enterprise Security Features
692
+
693
+ | Feature | Description |
694
+ |---------|-------------|
695
+ | **Audit Logging** | Action tracking for compliance |
696
+ | **Secure Exports** | Validated export endpoints |
697
+ | **File Type Validation** | Whitelist-based file type checking |
698
+ | **Size Limits** | 100MB per file upload |
699
+
700
+ ---
701
+
702
+ ## 10. Success Metrics & KPIs
703
+
704
+ ### 10.1 Product Metrics
705
+
706
+ | Metric | Description | Target |
707
+ |--------|-------------|--------|
708
+ | **Time to First Insight** | Time from upload to first analysis | < 2 minutes |
709
+ | **Model Training Time (Fast)** | Fast mode training duration | < 60 seconds |
710
+ | **Model Training Time (Ultra)** | Ultra mode training duration | < 10 minutes |
711
+ | **Dashboard Generation Time** | Time to generate dashboard | < 30 seconds |
712
+ | **Chat Response Time** | AI response latency | < 3 seconds |
713
+
714
+ ### 10.2 Quality Metrics
715
+
716
+ | Metric | Description | Target |
717
+ |--------|-------------|--------|
718
+ | **ML Model Accuracy** | Classification accuracy | > 85% |
719
+ | **Chat Answer Quality** | User satisfaction score | > 4.0/5.0 |
720
+ | **Report Completeness** | Coverage of key insights | > 90% |
721
+ | **Dashboard Relevance** | Chart appropriateness score | > 85% |
722
+
723
+ ### 10.3 Usage Metrics
724
+
725
+ | Metric | Description | Tracking |
726
+ |--------|-------------|----------|
727
+ | **DAU/MAU** | Active users | Analytics |
728
+ | **Files Uploaded** | Data engagement | Database |
729
+ | **Models Trained** | ML adoption | Database |
730
+ | **Reports Generated** | Reporting usage | Database |
731
+ | **Chat Queries** | AI engagement | Logs |
732
+
733
+ ### 10.4 Business Metrics
734
+
735
+ | Metric | Description | Target |
736
+ |--------|-------------|--------|
737
+ | **User Retention (D7)** | 7-day retention | > 40% |
738
+ | **User Retention (D30)** | 30-day retention | > 25% |
739
+ | **Feature Adoption** | % using AutoML | > 30% |
740
+ | **Session Duration** | Average session time | > 10 minutes |
741
+
742
+ ---
743
+
744
+ ## 11. Roadmap & Future Enhancements
745
+
746
+ ### 11.1 Current State (v1.0) ✅
747
+
748
+ | Feature | Status |
749
+ |---------|--------|
750
+ | Data Upload & Management | ✅ Complete |
751
+ | AutoML (Fast/Ultra) | ✅ Complete |
752
+ | ML Predictions & Playground | ✅ Complete |
753
+ | Autonomous Dashboard | ✅ Complete |
754
+ | AI Analyst Chat (7 Modes) | ✅ Complete |
755
+ | Reports Generation | ✅ Complete |
756
+ | User Authentication | ✅ Complete |
757
+ | Dark/Light Theme | ✅ Complete |
758
+
759
+ ### 11.2 Near-term Roadmap (Q1-Q2 2025)
760
+
761
+ | Feature | Priority | Description |
762
+ |---------|----------|-------------|
763
+ | **Database Connectors** | High | Direct SQL/PostgreSQL/MySQL connections |
764
+ | **API Data Sources** | High | REST API data ingestion |
765
+ | **Collaborative Workspaces** | Medium | Team sharing and collaboration |
766
+ | **Custom Dashboards** | Medium | User-designed dashboard layouts |
767
+ | **Scheduled Reports** | Medium | Email reports on schedule |
768
+ | **Model API Export** | Medium | REST API for trained models |
769
+
770
+ ### 11.3 Mid-term Roadmap (Q3-Q4 2025)
771
+
772
+ | Feature | Priority | Description |
773
+ |---------|----------|-------------|
774
+ | **Real-time Data Streams** | High | Kafka/streaming data support |
775
+ | **Advanced Forecasting** | High | Prophet, ARIMA, neural forecasting |
776
+ | **Natural Language to SQL** | Medium | Text-to-SQL queries |
777
+ | **Embedded Analytics** | Medium | Embed dashboards in other apps |
778
+ | **White-label Solution** | Low | Custom branding options |
779
+
780
+ ### 11.4 Long-term Vision (2026+)
781
+
782
+ | Feature | Description |
783
+ |---------|-------------|
784
+ | **AutoML v3** | Neural architecture search, AutoML for deep learning |
785
+ | **Real-time Anomaly Alerts** | Streaming anomaly detection |
786
+ | **Predictive Actions** | Auto-trigger actions based on predictions |
787
+ | **Multi-tenant Enterprise** | Full enterprise deployment |
788
+ | **On-premise Deployment** | Self-hosted enterprise version |
789
+
790
+ ---
791
+
792
+ ## 12. Appendix
793
+
794
+ ### 12.1 Glossary
795
+
796
+ | Term | Definition |
797
+ |------|------------|
798
+ | **AutoML** | Automated Machine Learning - auto model selection and tuning |
799
+ | **RAG** | Retrieval-Augmented Generation - AI with document context |
800
+ | **SHAP** | SHapley Additive exPlanations - model interpretability |
801
+ | **KPI** | Key Performance Indicator |
802
+ | **LLM** | Large Language Model |
803
+ | **FAISS** | Facebook AI Similarity Search - vector database |
804
+
805
+ ### 12.2 Technical Requirements
806
+
807
+ **Minimum Server Requirements:**
808
+ - CPU: 4 cores
809
+ - RAM: 8GB (16GB recommended)
810
+ - Storage: 50GB SSD
811
+ - Python: 3.11+
812
+ - Node.js: 18+
813
+
814
+ **Supported Browsers:**
815
+ - Chrome 90+
816
+ - Firefox 90+
817
+ - Safari 14+
818
+ - Edge 90+
819
+
820
+ ### 12.3 Contact & Support
821
+
822
+ | Resource | Link |
823
+ |----------|------|
824
+ | **Production URL** | https://killerkumar-ai-business-analyst.hf.space |
825
+ | **Documentation** | This PRD |
826
+ | **Support Email** | insights@ai20insights.tech |
827
+
828
+ ---
829
+
830
+ **Document History:**
831
+
832
+ | Version | Date | Author | Changes |
833
+ |---------|------|--------|---------|
834
+ | 1.0 | January 2025 | DataVision Team | Initial PRD |
835
+
836
+ ---
837
+
838
+ *© 2025 DataVision AI. All rights reserved.*
README.md ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: DataVision AI
3
+ emoji: 📊
4
+ colorFrom: blue
5
+ colorTo: indigo
6
+ sdk: docker
7
+ app_port: 7860
8
+ pinned: false
9
+ ---
10
+
11
+ <div align="center">
12
+ <img src="frontend/public/datavision-logo.png" alt="DataVision logo" width="96" />
13
+ <h1>DataVision AI</h1>
14
+ <p><strong>Autonomous analytics, business intelligence, AutoML, computer vision, and team collaboration in one platform.</strong></p>
15
+ <p>
16
+ <a href="https://datavision-ai-datavision.hf.space">Live application</a> ·
17
+ <a href="frontend/public/DataVision_AI_Product_User_Guide.pdf">Product user guide (PDF)</a>
18
+ </p>
19
+ </div>
20
+
21
+ ## What DataVision provides
22
+
23
+ - Interactive dashboards with slicers, KPI cards, business charts, AI explanations, exports, and themes.
24
+ - AI Analyst for grounded questions about uploaded datasets.
25
+ - AutoML, predictions, forecasts, reports, vector search, and data pipelines.
26
+ - Computer Vision for detection, classification, segmentation, pose estimation, and OCR.
27
+ - Persistent collaboration with channels, replies, reactions, pinning, and message deletion.
28
+ - Developer API keys, webhooks, usage reporting, and embed tools.
29
+
30
+ ## Run locally
31
+
32
+ Requirements: Python 3.11+, Node.js 20+, and PostgreSQL for persistent production data.
33
+
34
+ ```bash
35
+ # Backend
36
+ cd backend
37
+ python -m venv .venv
38
+ .venv\Scripts\activate
39
+ pip install -r requirements.txt
40
+ python main.py
41
+
42
+ # Frontend (in a second terminal)
43
+ cd frontend
44
+ npm install
45
+ npm run dev
46
+ ```
47
+
48
+ The frontend opens at `http://localhost:5173`.
49
+
50
+ ## Configuration
51
+
52
+ Create `backend/.env` with your production settings. Do not commit secrets.
53
+
54
+ ```env
55
+ DATABASE_URL=postgresql+asyncpg://user:password@host:5432/datavision
56
+ GROQ_API_KEY=your_key
57
+ JWT_SECRET=a_long_random_secret
58
+ ENVIRONMENT=development
59
+ CORS_ORIGINS=http://localhost:5173
60
+ ```
61
+
62
+ ## Deployment
63
+
64
+ Pushes to `main` run the GitHub Actions workflows in `.github/workflows/`. Configure these GitHub secrets before deployment:
65
+
66
+ - `HF_TOKEN` (Hugging Face write token)
67
+ - Optional `HF_SPACE` (defaults to `datavision-ai/Datavision`)
68
+
69
+ Set `DATABASE_URL`, `GROQ_API_KEY`, and `JWT_SECRET` in your Hugging Face Space settings. See [HUGGINGFACE_DEPLOYMENT.md](HUGGINGFACE_DEPLOYMENT.md) for the checklist.
70
+
71
+ ## User guide
72
+
73
+ The updated guide is available here: [DataVision AI Product User Guide](frontend/public/DataVision_AI_Product_User_Guide.pdf).
74
+
75
+ ## Validation
76
+
77
+ ```bash
78
+ cd frontend
79
+ npm run build
80
+ ```
81
+
82
+ ```bash
83
+ cd backend
84
+ python -m py_compile main.py api/v1/endpoints/collaboration.py
85
+ ```
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/Dockerfile ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.10-slim
2
+
3
+ WORKDIR /app
4
+
5
+ COPY requirements.txt .
6
+ RUN pip install --no-cache-dir -r requirements.txt
7
+
8
+ COPY . .
9
+
10
+ EXPOSE 5000
11
+
12
+ CMD ["python", "api_server.py", "--port", "5000"]
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/README.md ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚀 DataVision ML Project — Complete Guide
2
+
3
+ Auto-generated by **DataVision AI Business Analyst**
4
+ *This ZIP contains YOUR actual trained model, data, scripts, and everything needed to use it.*
5
+
6
+ ---
7
+
8
+ ## 📦 What's Inside This ZIP?
9
+
10
+ ```
11
+ ├── model.pkl # Your trained model
12
+ ├── data/
13
+ │ └── cleaned_data.csv # Your cleaned dataset
14
+ ├── predict.py # Make predictions (CLI / import)
15
+ ├── visualize.py # Generate performance charts
16
+ ├── evaluate.py # Comprehensive model evaluation
17
+ ├── train_traditional.py # Traditional ML training
18
+ ├── charts/ # Pre-generated training charts (PNG)
19
+ ├── api_server.py # Flask REST API for deployment
20
+ ├── utils/
21
+ │ └── preprocessing.py # Data preprocessing utilities
22
+ ├── config.json # Full training configuration
23
+ ├── requirements.txt # Python dependencies
24
+ ├── Dockerfile # Docker deployment
25
+ └── README.md # This file
26
+ ```
27
+
28
+ ### What Each File Does:
29
+
30
+ | File | What It Does | When To Use |
31
+ |------|-------------|-------------|
32
+ | `model.pkl` | Your **already trained** ML model (ready to use) | Loaded automatically by other scripts |
33
+ | `data/cleaned_data.csv` | Your cleaned dataset (used for charts & evaluation) | Used by visualize.py and evaluate.py |
34
+ | `predict.py` | Make predictions on new data | When you want to predict something |
35
+ | `visualize.py` | Generate all charts (confusion matrix, ROC, etc.) | When you want to see model performance charts |
36
+ | `evaluate.py` | Show detailed model metrics and scores | When you want to see accuracy, F1, R2 scores |
37
+ | `api_server.py` | Start a web API server with a form | When you want a web interface or REST API |
38
+ | `requirements.txt` | List of Python packages needed | Install once before running anything |
39
+ | `config.json` | Full training configuration and metadata | Reference only |
40
+ | `README.md` | This guide | You are reading it now |
41
+
42
+ ---
43
+
44
+ ## Step-by-Step: Getting Started (For Beginners)
45
+
46
+ ### STEP 1: Install Python (if not installed)
47
+
48
+ Download and install Python 3.8 or newer from https://www.python.org/downloads/
49
+
50
+ > **Important**: During installation, check the box **"Add Python to PATH"**
51
+
52
+ ### STEP 2: Open Terminal / Command Prompt
53
+
54
+ - **Windows**: Press `Win + R`, type `cmd`, press Enter
55
+ - **Mac/Linux**: Open Terminal app
56
+
57
+ Navigate to the project folder:
58
+ ```bash
59
+ cd path/to/this/folder
60
+ ```
61
+
62
+ ### STEP 3: Install Dependencies (Run Once)
63
+
64
+ ```bash
65
+ pip install -r requirements.txt
66
+ ```
67
+
68
+ This installs all the Python packages needed. You only need to do this **once**.
69
+
70
+ > If you get an error, try: `pip3 install -r requirements.txt`
71
+
72
+ ---
73
+
74
+ ## STEP 4: See Your Model Results (evaluate.py)
75
+
76
+ This shows your **exact same scores** from DataVision.
77
+ Automatically evaluates the correct model (Traditional ML or Deep Learning) based on your best training mode.
78
+
79
+ ```bash
80
+ python evaluate.py # Auto-detect best model
81
+ python evaluate.py --mode dl # Force Deep Learning model
82
+ python evaluate.py --mode traditional # Force traditional ML model
83
+ ```
84
+
85
+ **What you will see:**
86
+ - Your model accuracy or R2 score
87
+ - Classification report (precision, recall, F1 per class)
88
+ - Confusion matrix
89
+ - All stored metrics from training
90
+
91
+ **Additional options:**
92
+ ```bash
93
+ python evaluate.py --cv 5 # Run 5-fold cross-validation
94
+ python evaluate.py --reeval # Re-evaluate model on dataset
95
+ ```
96
+
97
+ **Expected output example:**
98
+ ```
99
+ Model: RandomForest
100
+ Task: classification
101
+ Accuracy: 0.9450
102
+ F1-Score: 0.9380
103
+ ```
104
+
105
+ ---
106
+
107
+ ## STEP 5: Make Predictions (predict.py)
108
+
109
+ ### Option A: Interactive Mode (Easiest)
110
+ ```bash
111
+ python predict.py
112
+ ```
113
+ It will ask you to enter a value for each feature one by one, then show the prediction.
114
+
115
+ ### Option B: JSON Input (Single Prediction)
116
+ ```bash
117
+ python predict.py --json "{\"feature1\": 25, \"feature2\": \"value\"}"
118
+ ```
119
+ Replace `feature1`, `feature2` with your actual column names.
120
+
121
+ ### Option C: CSV Batch Predictions (Many Rows)
122
+ ```bash
123
+ python predict.py --csv your_data.csv
124
+ ```
125
+ This reads your CSV, predicts for every row, saves results to `your_data_predictions.csv`.
126
+
127
+ ```bash
128
+ python predict.py --csv input.csv --output results.csv
129
+ ```
130
+
131
+ **Expected output example:**
132
+ ```
133
+ Prediction: Normal
134
+ Confidence: 94.5%
135
+ Probabilities:
136
+ Normal: 94.5%
137
+ Abnormal: 5.5%
138
+ ```
139
+
140
+ ---
141
+
142
+ ## STEP 6: Generate Charts (visualize.py)
143
+
144
+ Creates **all the same charts** you saw in DataVision AutoML.
145
+ Automatically loads the correct model (Traditional ML or Deep Learning) based on your best training mode.
146
+
147
+ ```bash
148
+ python visualize.py # Auto-detect best model
149
+ python visualize.py --mode dl # Force Deep Learning model
150
+ python visualize.py --mode traditional # Force traditional ML model
151
+ ```
152
+
153
+ Charts are saved to a `charts/` folder. Open them with any image viewer.
154
+
155
+ **Classification charts generated:**
156
+
157
+ | Chart File | What It Shows |
158
+ |------------|---------------|
159
+ | `model_comparison.png` | All models ranked by score |
160
+ | `feature_importance.png` | Which features matter most |
161
+ | `confusion_matrix.png` | Correct vs wrong predictions per class |
162
+ | `class_distribution.png` | Actual vs predicted class counts |
163
+ | `roc_curve.png` | ROC curve with AUC score |
164
+ | `precision_recall.png` | Precision-Recall curve |
165
+ | `calibration.png` | Probability calibration curve |
166
+ | `confidence_histogram.png` | Confidence for correct vs incorrect |
167
+ | `metrics_heatmap.png` | Per-class precision, recall, F1 |
168
+ | `class_balance_pie.png` | Class balance pie charts |
169
+ | `confidence_boxplot.png` | Confidence box plot per class |
170
+ | `score_distribution.png` | Score distribution per class |
171
+
172
+ **Regression charts generated:**
173
+
174
+ | Chart File | What It Shows |
175
+ |------------|---------------|
176
+ | `actual_vs_predicted.png` | Scatter: actual vs predicted values |
177
+ | `residuals_analysis.png` | 4-panel residual analysis |
178
+ | `error_distribution.png` | Error distribution with percentiles |
179
+ | `prediction_overview.png` | Sorted predictions with error band |
180
+ | `error_boxplot.png` | Error by target range (Q1-Q4) |
181
+ | `scale_location.png` | Scale-Location diagnostic plot |
182
+
183
+ **Common charts (both):**
184
+
185
+ | Chart File | What It Shows |
186
+ |------------|---------------|
187
+ | `correlation_heatmap.png` | Feature correlations |
188
+ | `distribution_grid.png` | Distribution of each feature |
189
+ | `boxplot_grid.png` | Outlier detection per feature |
190
+ | `target_distribution.png` | Target column distribution |
191
+
192
+ **Deep Learning charts (when DL model is loaded):**
193
+
194
+ | Chart File | What It Shows |
195
+ |------------|---------------|
196
+ | `training_history.png` | Training loss curve and validation scores |
197
+ | `dl_model_summary.png` | Model architecture, hyperparameters, metrics |
198
+ | `feature_importance.png` | MLP weight-based feature importance |
199
+
200
+ **Unsupervised Learning charts (auto-generated):**
201
+
202
+ | Chart File | What It Shows |
203
+ |------------|---------------|
204
+ | `cluster_scatter.png` | KMeans clustering with PCA 2D projection |
205
+ | `cluster_distribution.png` | Cluster size distribution |
206
+
207
+ > **Note:** Your ZIP already contains pre-generated charts in the `charts/` folder
208
+ > from your actual training. These are the EXACT same charts you saw in DataVision.
209
+ > Running `visualize.py` will regenerate them from your model data.
210
+
211
+ **Custom options:**
212
+ ```bash
213
+ python visualize.py --data other_data.csv # Use different data
214
+ python visualize.py --output my_charts/ # Save to different folder
215
+ ```
216
+
217
+ ---
218
+
219
+ ## STEP 7: Deploy as Web API (api_server.py)
220
+
221
+ Start a local web server with a prediction form:
222
+
223
+ ```bash
224
+ python api_server.py
225
+ ```
226
+
227
+ Then open your browser and go to: **http://localhost:5000/predict**
228
+
229
+ You will see a **web form** with your actual column names where you can:
230
+ 1. Enter values for each feature
231
+ 2. Click "Predict"
232
+ 3. See the prediction result instantly
233
+
234
+ **API Endpoints:**
235
+
236
+ | URL | Method | What It Does |
237
+ |-----|--------|-------------|
238
+ | `/predict` | GET | Web form with your feature names |
239
+ | `/predict` | POST | JSON API for developers |
240
+ | `/health` | GET | Health check (is server running?) |
241
+ | `/model-info` | GET | Model metadata and metrics |
242
+
243
+ **Change port:**
244
+ ```bash
245
+ python api_server.py --port 8080
246
+ ```
247
+
248
+ ---
249
+
250
+ ## STEP 8: Retrain the Model (Optional)
251
+
252
+ > Your model is already trained in `model.pkl`. Only retrain if you want to
253
+ > experiment with new data or verify the training.
254
+
255
+ #### Traditional ML
256
+ ```bash
257
+ python train_traditional.py # Reproduces your training
258
+ python train_traditional.py --data your.csv # Custom data
259
+ ```
260
+
261
+ ---
262
+
263
+ ## STEP 9: Docker Deployment (Advanced)
264
+
265
+ ```bash
266
+ docker build -t my-ml-model .
267
+ docker run -p 5000:5000 my-ml-model
268
+ ```
269
+
270
+ Then access: `http://localhost:5000/predict`
271
+
272
+ ---
273
+
274
+ ## Your Model Summary
275
+
276
+ | Property | Value |
277
+ |----------|-------|
278
+ | **Target Column** | `price_range` |
279
+ | **Task Type** | classification |
280
+ | **Best Model** | RandomForest |
281
+ | **Training Mode** | traditional |
282
+ | **Features** | 20 columns |
283
+ | **Dataset Rows** | 2000 |
284
+ | **Modes Trained** | traditional |
285
+
286
+ ### Training Metrics
287
+ - **ACCURACY**: 0.8925 (89.2%)
288
+ - **F1**: 0.8929 (89.3%)
289
+ - **PRECISION**: 0.8937 (89.4%)
290
+ - **RECALL**: 0.8925 (89.2%)
291
+ - **ROC_AUC**: 0.9796 (98.0%)
292
+
293
+ ### Model Leaderboard
294
+
295
+ | Rank | Model | Score |
296
+ |------|-------|-------|
297
+ | 1 | ? | 0.8925 |
298
+
299
+ ### Feature Details
300
+
301
+ | Feature | Type | Range/Values |
302
+ |---------|------|-------------|
303
+ | `battery_power` | numeric | 510.0 - 1987.0 |
304
+ | `blue` | numeric | 0.0 - 1.0 |
305
+ | `clock_speed` | numeric | 0.5 - 3.0 |
306
+ | `dual_sim` | numeric | 0.0 - 1.0 |
307
+ | `fc` | numeric | 0.0 - 16.0 |
308
+ | `four_g` | numeric | 0.0 - 1.0 |
309
+ | `int_memory` | numeric | 2.0 - 64.0 |
310
+ | `m_dep` | numeric | 0.1 - 1.0 |
311
+ | `mobile_wt` | numeric | 80.0 - 199.0 |
312
+ | `n_cores` | numeric | 1.0 - 8.0 |
313
+
314
+ ---
315
+
316
+ ## Troubleshooting
317
+
318
+ | Problem | Solution |
319
+ |---------|----------|
320
+ | `ModuleNotFoundError` | Run `pip install -r requirements.txt` |
321
+ | `model.pkl not found` | Make sure you are in the project folder |
322
+ | `No module named flask` | Run `pip install flask` |
323
+ | `Port already in use` | Use `python api_server.py --port 8080` |
324
+ | Charts not generating | Make sure `data/cleaned_data.csv` exists |
325
+ | Wrong model loaded | Use `--mode dl` or `--mode traditional` to force specific model |
326
+ | DL model not found | Make sure `deep_learning_model.pkl` exists |
327
+ | Permission denied | On Mac/Linux: try `python3` instead of `python` |
328
+
329
+ ---
330
+
331
+ ## Using as Python Module (For Developers)
332
+
333
+ ```python
334
+ from predict import load_model, predict_single
335
+
336
+ state = load_model('model.pkl')
337
+ result = predict_single(state, {
338
+ # Use your actual column names here
339
+ # Run 'python predict.py' to see what columns are needed
340
+ })
341
+ print(f"Prediction: {result['prediction']}")
342
+ if 'confidence' in result:
343
+ print(f"Confidence: {result['confidence']:.1%}")
344
+ ```
345
+
346
+ ---
347
+ *Generated by DataVision AI Business Analyst*
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/api_server.py ADDED
@@ -0,0 +1,633 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🌐 Flask API Server — Your Trained Model
3
+ ============================================
4
+ Auto-generated by DataVision AI
5
+
6
+ Model : RandomForest
7
+ Target Column : price_range
8
+ Task Type : classification
9
+ Features : 20 columns
10
+
11
+ Usage:
12
+ python api_server.py
13
+ python api_server.py --port 8080
14
+
15
+ Endpoints:
16
+ POST /predict — Make a prediction (JSON body with your feature names)
17
+ GET /health — Health check
18
+ GET /model-info — Model metadata and expected features
19
+ """
20
+
21
+ import os
22
+ import json
23
+ import re
24
+ import pickle
25
+ import argparse
26
+ import numpy as np
27
+ import pandas as pd
28
+ from flask import Flask, request, jsonify
29
+ from sklearn.preprocessing import LabelEncoder, RobustScaler
30
+
31
+ app = Flask(__name__)
32
+
33
+ MODEL_STATE = None
34
+ FEATURE_ENGINEER = None
35
+ FEATURE_COLUMNS = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt', 'n_cores', 'pc', 'px_height', 'px_width', 'ram', 'sc_h', 'sc_w', 'talk_time', 'three_g', 'touch_screen', 'wifi']
36
+ TARGET_COLUMN = 'price_range'
37
+ BEST_MODE = 'traditional' # Which mode won during AutoML
38
+
39
+ # YOUR original column names with types and ranges (what you input)
40
+ RAW_FEATURE_INFO = {
41
+ "battery_power": {"type": "numeric", "min": 510.0, "max": 1987.0, "default": 1238.51},
42
+ "blue": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.49},
43
+ "clock_speed": {"type": "numeric", "min": 0.5, "max": 3.0, "default": 1.52},
44
+ "dual_sim": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
45
+ "fc": {"type": "numeric", "min": 0.0, "max": 16.0, "default": 4.29},
46
+ "four_g": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.52},
47
+ "int_memory": {"type": "numeric", "min": 2.0, "max": 64.0, "default": 32.05},
48
+ "m_dep": {"type": "numeric", "min": 0.1, "max": 1.0, "default": 0.50},
49
+ "mobile_wt": {"type": "numeric", "min": 80.0, "max": 199.0, "default": 140.24},
50
+ "n_cores": {"type": "numeric", "min": 1.0, "max": 8.0, "default": 4.52},
51
+ "pc": {"type": "numeric", "min": 0.0, "max": 20.0, "default": 9.92},
52
+ "px_height": {"type": "numeric", "min": 15.0, "max": 1791.01, "default": 644.46},
53
+ "ram": {"type": "numeric", "min": 296.99, "max": 3958.01, "default": 2124.26},
54
+ "sc_h": {"type": "numeric", "min": 5.0, "max": 19.0, "default": 12.31},
55
+ "sc_w": {"type": "numeric", "min": 0.0, "max": 17.0, "default": 5.76},
56
+ "three_g": {"type": "numeric", "min": 1.0, "max": 1.0, "default": 1.00},
57
+ "touch_screen": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.50},
58
+ "wifi": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
59
+ }
60
+
61
+
62
+ # =============================================================================
63
+ # Standalone Feature Engineer — exact DataVision preprocessing
64
+ # =============================================================================
65
+ class StandaloneFeatureEngineer:
66
+ def __init__(self, state):
67
+ self.transformers = state.get('transformers', {})
68
+ self.encoders = state.get('encoders', {})
69
+ self.feature_names = state.get('feature_names', [])
70
+ self.original_columns = state.get('original_columns', [])
71
+ self.selected_feature_indices = state.get('selected_feature_indices', None)
72
+ self.is_fitted = state.get('is_fitted', True)
73
+ self.stopwords = set("i me my myself we our ours ourselves you your yours yourself yourselves he him his himself she her hers herself it its itself they them their theirs themselves what which who whom this that these those am is are was were be been being have has had having do does did doing a an the and but if or because as until while of at by for with about against between through during before after above below to from up down in out on off over under again further then once here there when where why how all both each few more most other some such no nor not only own same so than too very s t can will just don should now d ll m o re ve y ain aren couldn didn doesn hadn hasn haven isn ma mightn mustn needn shan shouldn wasn weren won wouldn".split())
74
+ self.positive_words = set("good great excellent amazing wonderful fantastic brilliant outstanding superb love best perfect beautiful awesome incredible impressive magnificent exceptional fabulous terrific marvelous stellar remarkable extraordinary delightful gorgeous splendid spectacular phenomenal exquisite lovely charming pleasant nice fine happy joy enjoy satisfied pleased glad cheerful excited thrilled".split())
75
+ self.negative_words = set("bad terrible horrible awful worst poor disappointing disgusting pathetic useless dreadful atrocious appalling abysmal mediocre inferior unpleasant nasty vile wretched miserable lousy horrendous hideous ghastly hideous deplorable lamentable pitiful woeful tragic sad angry frustrated annoyed irritated disappointed upset unhappy distressed sorrowful gloomy depressed".split())
76
+
77
+ def _clean_text(self, text):
78
+ text = str(text).lower()
79
+ text = re.sub(r'<[^>]+>', ' ', text)
80
+ text = re.sub(r'http\S+', ' ', text)
81
+ text = re.sub(r'[^a-z0-9\s]', ' ', text)
82
+ text = ' '.join(w for w in text.split() if w not in self.stopwords and len(w) > 1)
83
+ return text if text.strip() else 'empty'
84
+
85
+ def _text_statistics(self, text):
86
+ text = str(text)
87
+ words = text.split()
88
+ wl = [len(w) for w in words] if words else [0]
89
+ return [len(text), len(words), np.mean(wl), np.std(wl), max(wl) if wl else 0,
90
+ len(set(words)) / max(len(words), 1), text.count('!'), text.count('?'),
91
+ sum(1 for c in text if c.isupper()) / max(len(text), 1),
92
+ len([w for w in words if len(w) > 6]) / max(len(words), 1)]
93
+
94
+ def _sentiment_score(self, text):
95
+ words = set(str(text).lower().split())
96
+ pos = len(words & self.positive_words)
97
+ neg = len(words & self.negative_words)
98
+ total = max(pos + neg, 1)
99
+ return [pos / total, neg / total, (pos - neg) / total, total]
100
+
101
+ def transform(self, df):
102
+ df = df.copy()
103
+ cols = df.columns.tolist()
104
+ seen = {}
105
+ new_cols = []
106
+ for col in cols:
107
+ if col in seen:
108
+ seen[col] += 1
109
+ new_cols.append(f"{col}_{seen[col]}")
110
+ else:
111
+ seen[col] = 0
112
+ new_cols.append(col)
113
+ df.columns = new_cols
114
+ feature_parts = []
115
+
116
+ if 'numeric_cols' in self.transformers:
117
+ numeric_cols = self.transformers['numeric_cols']
118
+ for col in numeric_cols:
119
+ if col not in df.columns:
120
+ df[col] = 0
121
+ try:
122
+ numeric_data = df[numeric_cols].apply(pd.to_numeric, errors='coerce').fillna(0).values
123
+ scaler = self.transformers['numeric_scaler']
124
+ numeric_scaled = scaler.transform(numeric_data)
125
+ numeric_scaled = np.nan_to_num(numeric_scaled, nan=0.0, posinf=0.0, neginf=0.0)
126
+ feature_parts.append(numeric_scaled)
127
+ if 2 <= len(numeric_cols) <= 10:
128
+ interactions = []
129
+ for i in range(min(len(numeric_cols), 4)):
130
+ interactions.append((numeric_scaled[:, i] ** 2).reshape(-1, 1))
131
+ for i in range(min(len(numeric_cols), 4)):
132
+ for j in range(i + 1, min(len(numeric_cols), 4)):
133
+ interactions.append((numeric_scaled[:, i] * numeric_scaled[:, j]).reshape(-1, 1))
134
+ if interactions:
135
+ feature_parts.append(np.hstack(interactions))
136
+ except:
137
+ feature_parts.append(np.zeros((len(df), len(numeric_cols))))
138
+
139
+ for key, cols in self.transformers.items():
140
+ if key.endswith('_onehot'):
141
+ col_name = key.replace('_onehot', '')
142
+ try:
143
+ if col_name in df.columns:
144
+ series = df[col_name].fillna('_MISSING_').astype(str).str.strip()
145
+ dummies = pd.get_dummies(series, prefix=col_name)
146
+ dummies = dummies.reindex(columns=cols, fill_value=0)
147
+ feature_parts.append(dummies.values.astype(float))
148
+ else:
149
+ feature_parts.append(np.zeros((len(df), len(cols))))
150
+ except:
151
+ feature_parts.append(np.zeros((len(df), len(cols))))
152
+
153
+ for col, le in self.encoders.items():
154
+ try:
155
+ if col in df.columns:
156
+ series = df[col].fillna('_MISSING_').astype(str).str.strip()
157
+ encoded = series.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
158
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
159
+ freq_key = f'{col}_freq_map'
160
+ if freq_key in self.transformers:
161
+ freq_map = self.transformers[freq_key]
162
+ freq_encoded = series.map(lambda s: freq_map.get(s, 0.0)).values.astype(float)
163
+ feature_parts.append(freq_encoded.reshape(-1, 1))
164
+ else:
165
+ feature_parts.append(np.zeros((len(df), 1)))
166
+ if f'{col}_freq_map' in self.transformers:
167
+ feature_parts.append(np.zeros((len(df), 1)))
168
+ except:
169
+ feature_parts.append(np.zeros((len(df), 1)))
170
+
171
+ if 'cat_interactions' in self.transformers:
172
+ for key, value in self.transformers['cat_interactions'].items():
173
+ if key.endswith('_freq_map'):
174
+ continue
175
+ try:
176
+ col1, col2, le = value
177
+ if col1 in df.columns and col2 in df.columns:
178
+ combined = df[col1].fillna('_NA_').astype(str) + "_X_" + df[col2].fillna('_NA_').astype(str)
179
+ encoded = combined.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
180
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
181
+ fk = f"{key}_freq_map"
182
+ if fk in self.transformers['cat_interactions']:
183
+ fm = self.transformers['cat_interactions'][fk]
184
+ feature_parts.append(combined.map(lambda s: fm.get(s, 0.0)).values.reshape(-1, 1).astype(float))
185
+ else:
186
+ feature_parts.append(np.zeros((len(df), 1)))
187
+ except:
188
+ pass
189
+
190
+ if 'num_cat_aggs' in self.transformers:
191
+ for key, mean_map in self.transformers['num_cat_aggs'].items():
192
+ try:
193
+ parts = key.rsplit('_by_', 1)
194
+ if len(parts) == 2:
195
+ num_col, cat_col = parts[0], parts[1].rsplit('_mean', 1)[0]
196
+ if num_col in df.columns and cat_col in df.columns:
197
+ cat_s = df[cat_col].fillna('_NA_').astype(str)
198
+ dv = np.mean(list(mean_map.values())) if mean_map else 0
199
+ agg = cat_s.map(lambda s: mean_map.get(s, dv)).values.astype(float)
200
+ feature_parts.append(agg.reshape(-1, 1))
201
+ feature_parts.append((df[num_col].fillna(0).values - agg).reshape(-1, 1))
202
+ else:
203
+ feature_parts.extend([np.zeros((len(df), 1))] * 2)
204
+ except:
205
+ pass
206
+
207
+ for key, tfidf in self.transformers.items():
208
+ if key.endswith('_tfidf'):
209
+ col = key.replace('_tfidf', '')
210
+ try:
211
+ if col in df.columns:
212
+ series = df[col].fillna('').astype(str)
213
+ cleaned = series.apply(self._clean_text)
214
+ tm = tfidf.transform(cleaned)
215
+ svd_key = f'{col}_svd'
216
+ feature_parts.append(self.transformers[svd_key].transform(tm) if svd_key in self.transformers else tm.toarray())
217
+ feature_parts.append(np.array([self._text_statistics(t) for t in series]))
218
+ feature_parts.append(np.array([self._sentiment_score(t) for t in series]))
219
+ except:
220
+ pass
221
+
222
+ if not feature_parts:
223
+ return np.zeros((len(df), len(self.feature_names)))
224
+ X = np.hstack(feature_parts)
225
+ X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
226
+
227
+ if 'variance_selector' in self.transformers:
228
+ sel = self.transformers['variance_selector']
229
+ try:
230
+ if X.shape[1] == sel.n_features_in_:
231
+ X = sel.transform(X)
232
+ elif self.selected_feature_indices is not None:
233
+ vi = [i for i in self.selected_feature_indices if i < X.shape[1]]
234
+ if vi:
235
+ X = X[:, vi]
236
+ except:
237
+ pass
238
+
239
+ if 'pca' in self.transformers:
240
+ try:
241
+ pca = self.transformers['pca']
242
+ if X.shape[1] == pca.n_features_in_:
243
+ X = pca.transform(X)
244
+ except:
245
+ pass
246
+
247
+ if X.shape[1] != len(self.feature_names):
248
+ if X.shape[1] < len(self.feature_names):
249
+ X = np.hstack([X, np.zeros((len(df), len(self.feature_names) - X.shape[1]))])
250
+ else:
251
+ X = X[:, :len(self.feature_names)]
252
+ return X
253
+
254
+ def transform_single(self, data):
255
+ for col in self.original_columns:
256
+ if col not in data:
257
+ data[col] = 0
258
+ df = pd.DataFrame([data])
259
+ X = self.transform(df)
260
+ n = len(self.feature_names)
261
+ if X.shape[1] < n:
262
+ X = np.hstack([X, np.zeros((1, n - X.shape[1]))])
263
+ elif X.shape[1] > n:
264
+ X = X[:, :n]
265
+ return X
266
+
267
+
268
+ def load_model():
269
+ global MODEL_STATE, FEATURE_ENGINEER
270
+
271
+ # If best mode is deep_learning, load DL first as primary
272
+ if BEST_MODE == 'deep_learning':
273
+ load_dl_model()
274
+ if DL_STATE is not None:
275
+ print(f"🧠 Primary mode: Deep Learning (BEST_MODE={BEST_MODE})")
276
+ # Also try loading ML model as fallback
277
+ for path in ['model.pkl', 'nlp_model.pkl']:
278
+ if os.path.exists(path):
279
+ with open(path, 'rb') as f:
280
+ MODEL_STATE = pickle.load(f)
281
+ print(f" Also loaded fallback: {path}")
282
+ break
283
+ return True
284
+
285
+ # Traditional/NLP: load ML model first
286
+ for path in ['model.pkl', 'nlp_model.pkl']:
287
+ if os.path.exists(path):
288
+ with open(path, 'rb') as f:
289
+ MODEL_STATE = pickle.load(f)
290
+ print(f"✅ Loaded: {path} ({MODEL_STATE.get('model_name', 'Unknown')})")
291
+ # Load feature engineer for exact preprocessing
292
+ eng_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'feature_engineer.pkl')
293
+ if os.path.exists(eng_path):
294
+ try:
295
+ with open(eng_path, 'rb') as ef:
296
+ eng_state = pickle.load(ef)
297
+ FEATURE_ENGINEER = StandaloneFeatureEngineer(eng_state)
298
+ print(f" Pipeline: feature_engineer.pkl ({len(FEATURE_ENGINEER.feature_names)} features)")
299
+ except Exception as e:
300
+ print(f" Warning: Could not load feature_engineer.pkl: {e}")
301
+ break
302
+
303
+ # Also load DL model if available
304
+ load_dl_model()
305
+
306
+ if MODEL_STATE is None and DL_STATE is None:
307
+ print("❌ No model file found")
308
+ return False
309
+ return True
310
+
311
+
312
+ # =============================================================================
313
+ # Deep Learning Model Support (sklearn MLPClassifier/MLPRegressor)
314
+ # =============================================================================
315
+ DL_STATE = None
316
+
317
+ def load_dl_model():
318
+ """Load the Deep Learning model if available."""
319
+ global DL_STATE
320
+ dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deep_learning_model.pkl')
321
+ if os.path.exists(dl_path):
322
+ try:
323
+ with open(dl_path, 'rb') as f:
324
+ DL_STATE = pickle.load(f)
325
+ print(f"✅ Loaded DL model: {DL_STATE.get('algorithm', 'MLP')}")
326
+ except Exception as e:
327
+ print(f"⚠️ Failed to load DL model: {e}")
328
+
329
+
330
+ def preprocess_dl_input(data):
331
+ """Preprocess input for Deep Learning model using feature_metadata."""
332
+ feature_metadata = DL_STATE.get('feature_metadata', [])
333
+ feature_values = []
334
+
335
+ # Process numeric features
336
+ for meta in feature_metadata:
337
+ if meta.get('type') == 'numeric':
338
+ col = meta['name']
339
+ if col in data:
340
+ try:
341
+ feature_values.append(float(data[col]))
342
+ except (ValueError, TypeError):
343
+ feature_values.append(meta.get('mean', 0))
344
+ else:
345
+ feature_values.append(meta.get('mean', 0))
346
+
347
+ # Process categorical features (one-hot)
348
+ for meta in feature_metadata:
349
+ if meta.get('type') == 'categorical':
350
+ col = meta['name']
351
+ value = str(data.get(col, ''))
352
+ options = meta.get('options', [])
353
+ for opt in options:
354
+ feature_values.append(1.0 if value == opt else 0.0)
355
+ feature_values.append(1.0 if not value or value == 'nan' else 0.0)
356
+
357
+ # Fallback
358
+ if not feature_values and DL_STATE.get('feature_columns'):
359
+ for col in DL_STATE['feature_columns']:
360
+ if col in data:
361
+ try:
362
+ feature_values.append(float(data[col]) if not isinstance(data[col], str) else 0)
363
+ except:
364
+ feature_values.append(0)
365
+ else:
366
+ feature_values.append(0)
367
+
368
+ X = np.array([feature_values])
369
+
370
+ scaler = DL_STATE.get('scaler')
371
+ if scaler and hasattr(scaler, 'n_features_in_'):
372
+ expected = scaler.n_features_in_
373
+ actual = X.shape[1]
374
+ if actual < expected:
375
+ X = np.hstack([X, np.zeros((1, expected - actual))])
376
+ elif actual > expected:
377
+ X = X[:, :expected]
378
+
379
+ if scaler:
380
+ X = scaler.transform(X)
381
+
382
+ return X
383
+
384
+
385
+ def preprocess_input(data):
386
+ """Preprocess raw input using the EXACT same pipeline as DataVision AutoML."""
387
+ global FEATURE_ENGINEER
388
+
389
+ # Use feature engineer (exact AutoML pipeline)
390
+ if FEATURE_ENGINEER is not None:
391
+ try:
392
+ return FEATURE_ENGINEER.transform_single(data)
393
+ except Exception as e:
394
+ print(f"Warning: Feature engineer failed ({e}), using fallback")
395
+
396
+ # Fallback: basic label-encode + scale
397
+ feature_cols = MODEL_STATE.get('feature_columns', FEATURE_COLUMNS)
398
+ label_encoders = MODEL_STATE.get('label_encoders', {})
399
+ scaler = MODEL_STATE.get('scaler')
400
+
401
+ row = {}
402
+ for col in feature_cols:
403
+ val = data.get(col, 0)
404
+ if col in label_encoders:
405
+ le = label_encoders[col]
406
+ val_str = str(val)
407
+ if val_str in le.classes_:
408
+ val = le.transform([val_str])[0]
409
+ else:
410
+ val = 0
411
+ else:
412
+ try:
413
+ val = float(val)
414
+ except (ValueError, TypeError):
415
+ val = 0
416
+ row[col] = val
417
+
418
+ X = pd.DataFrame([row], columns=feature_cols)
419
+ if scaler:
420
+ try:
421
+ X = pd.DataFrame(scaler.transform(X), columns=feature_cols)
422
+ except:
423
+ pass
424
+ return X
425
+
426
+
427
+ @app.route('/health', methods=['GET'])
428
+ def health():
429
+ return jsonify({"status": "healthy", "model_loaded": MODEL_STATE is not None or DL_STATE is not None, "dl_model_loaded": DL_STATE is not None})
430
+
431
+
432
+ @app.route('/model-info', methods=['GET'])
433
+ def model_info():
434
+ if MODEL_STATE is None and DL_STATE is None:
435
+ return jsonify({"error": "No model loaded"}), 404
436
+
437
+ info = {"best_mode": BEST_MODE}
438
+ if BEST_MODE == 'deep_learning' and DL_STATE:
439
+ info.update({
440
+ "model_name": DL_STATE.get('algorithm', 'MLP Neural Network'),
441
+ "task_type": DL_STATE.get('task_type', 'classification'),
442
+ "target_column": TARGET_COLUMN,
443
+ "input_features": list(RAW_FEATURE_INFO.keys()),
444
+ "metrics": {k: float(v) if isinstance(v, (int, float)) else str(v)
445
+ for k, v in DL_STATE.get('metrics', {}).items()},
446
+ })
447
+ elif MODEL_STATE:
448
+ info.update({
449
+ "model_name": MODEL_STATE.get('model_name', 'Unknown'),
450
+ "task_type": MODEL_STATE.get('task_type', 'classification'),
451
+ "target_column": TARGET_COLUMN,
452
+ "input_features": list(RAW_FEATURE_INFO.keys()),
453
+ "metrics": {k: float(v) if isinstance(v, (int, float)) else str(v)
454
+ for k, v in MODEL_STATE.get('metrics', {}).items()},
455
+ })
456
+ if DL_STATE:
457
+ info["dl_model"] = {
458
+ "algorithm": DL_STATE.get('algorithm', 'MLP'),
459
+ "task_type": DL_STATE.get('task_type', 'classification'),
460
+ "metrics": {k: float(v) if isinstance(v, (int, float)) else str(v)
461
+ for k, v in DL_STATE.get('metrics', {}).items()},
462
+ }
463
+ return jsonify(info)
464
+
465
+
466
+ @app.route('/predict', methods=['GET', 'POST'])
467
+ def predict():
468
+ if MODEL_STATE is None and DL_STATE is None:
469
+ return jsonify({"error": "No model loaded"}), 500
470
+
471
+ # GET request: show web form for interactive prediction
472
+ if request.method == 'GET':
473
+ if BEST_MODE == 'deep_learning' and DL_STATE:
474
+ model_name = DL_STATE.get('algorithm', 'MLP Neural Network')
475
+ task_type_val = DL_STATE.get('task_type', 'classification')
476
+ metrics = DL_STATE.get('metrics', {})
477
+ elif MODEL_STATE:
478
+ model_name = MODEL_STATE.get('model_name', 'Unknown')
479
+ task_type_val = MODEL_STATE.get('task_type', 'classification')
480
+ metrics = MODEL_STATE.get('metrics', {})
481
+ else:
482
+ model_name = 'Unknown'
483
+ task_type_val = 'classification'
484
+ metrics = {}
485
+
486
+ # Build form from RAW features (original column names user knows)
487
+ form_fields = ""
488
+ for col, info in RAW_FEATURE_INFO.items():
489
+ ftype = info.get("type", "numeric")
490
+ if ftype == "categorical":
491
+ options = info.get("options", [])
492
+ opts_html = "".join('<option value="' + str(o) + '">' + str(o) + '</option>' for o in options[:20])
493
+ form_fields += '<div style="margin:6px 0"><label style="display:inline-block;width:260px;font-weight:500">' + col + '</label><select name="' + col + '" style="padding:6px 10px;border:1px solid #d1d5db;border-radius:6px;width:214px">' + opts_html + '</select></div>' + chr(10)
494
+ else:
495
+ default_val = str(info.get("default", 0))
496
+ mn = str(info.get("min", ""))
497
+ mx = str(info.get("max", ""))
498
+ placeholder = (mn + " - " + mx) if mn != "" else ""
499
+ form_fields += '<div style="margin:6px 0"><label style="display:inline-block;width:260px;font-weight:500">' + col + '</label><input name="' + col + '" type="text" value="' + default_val + '" placeholder="' + placeholder + '" style="padding:6px 10px;border:1px solid #d1d5db;border-radius:6px;width:200px"></div>' + chr(10)
500
+
501
+ metrics_html = ""
502
+ for k, v in metrics.items():
503
+ if isinstance(v, float):
504
+ metrics_html += f"<li><b>{k}</b>: {v:.4f}</li>"
505
+
506
+ html_tpl = '<!DOCTYPE html>'
507
+ html_tpl += '<html><head><title>DataVision ML API</title>'
508
+ html_tpl += '<style>'
509
+ html_tpl += "body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; max-width: 800px; margin: 40px auto; padding: 0 20px; background: #f9fafb; color: #111827; }}"
510
+ html_tpl += 'h1 {{ color: #4F46E5; }} h2 {{ color: #1f2937; border-bottom: 2px solid #e5e7eb; padding-bottom: 8px; }}'
511
+ html_tpl += '.card {{ background: white; border-radius: 12px; padding: 24px; margin: 16px 0; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }}'
512
+ html_tpl += 'button {{ background: #4F46E5; color: white; border: none; padding: 12px 32px; border-radius: 8px; font-size: 16px; cursor: pointer; margin-top: 12px; }}'
513
+ html_tpl += 'button:hover {{ background: #4338CA; }}'
514
+ html_tpl += '#result {{ margin-top: 20px; padding: 16px; background: #f0fdf4; border-radius: 8px; display: none; }}'
515
+ html_tpl += '</style></head><body>'
516
+ html_tpl += '<h1>🚀 DataVision ML API</h1>'
517
+ html_tpl += '<div class="card"><h2>Model Info</h2>'
518
+ html_tpl += '<p>🏆 <b>Model:</b> ' + str(model_name) + ' | 🎯 <b>Target:</b> price_range | 📊 <b>Task:</b> ' + str(task_type_val) + ' | 📁 <b>Features:</b> ' + str(len(RAW_FEATURE_INFO)) + '</p>'
519
+ html_tpl += '<ul>' + metrics_html + '</ul></div>'
520
+ html_tpl += '<div class="card"><h2>🔮 Make a Prediction</h2>'
521
+ html_tpl += '<form id="predForm">' + form_fields + '<button type="submit">Predict</button></form>'
522
+ html_tpl += '<div id="result"></div></div>'
523
+ html_tpl += '<div class="card"><h2>📡 API Usage</h2>'
524
+ html_tpl += '<pre style="background:#1f2937;color:#e5e7eb;padding:16px;border-radius:8px;overflow-x:auto">'
525
+ html_tpl += 'POST /predict\nContent-Type: application/json\n\n' + chr(123) + '"feature1": val, "feature2": val' + chr(125)
526
+ html_tpl += '\n\nGET /health — Health check\nGET /model-info — Model metadata</pre></div>'
527
+ html_tpl += '<script>'
528
+ html_tpl += 'document.getElementById("predForm").onsubmit=function(e)' + chr(123)
529
+ html_tpl += 'e.preventDefault();var fd=new FormData(this);var data=' + chr(123) + chr(125) + ';'
530
+ html_tpl += 'fd.forEach(function(v,k)' + chr(123) + 'data[k]=isNaN(v)||v===""?v:parseFloat(v)' + chr(125) + ');'
531
+ html_tpl += 'fetch("/predict",' + chr(123) + 'method:"POST",headers:' + chr(123) + '"Content-Type":"application/json"' + chr(125) + ',body:JSON.stringify(data)' + chr(125) + ')'
532
+ html_tpl += '.then(function(r)' + chr(123) + 'return r.json()' + chr(125) + ')'
533
+ html_tpl += '.then(function(j)' + chr(123)
534
+ html_tpl += 'var h="<h3>🎯 Prediction: "+j.prediction+"</h3>";'
535
+ html_tpl += 'if(j.confidence)h+="<p>📊 Confidence: "+(j.confidence*100).toFixed(1)+"%</p>";'
536
+ html_tpl += 'if(j.probabilities)' + chr(123) + 'h+="<p>Probabilities:</p><ul>";'
537
+ html_tpl += 'var ent=Object.entries(j.probabilities).sort(function(a,b)' + chr(123) + 'return b[1]-a[1]' + chr(125) + ');'
538
+ html_tpl += 'for(var i=0;i<ent.length;i++)h+="<li><b>"+ent[i][0]+"</b>: "+(ent[i][1]*100).toFixed(1)+"%</li>";'
539
+ html_tpl += 'h+="</ul>"' + chr(125)
540
+ html_tpl += 'document.getElementById("result").innerHTML=h;document.getElementById("result").style.display="block"'
541
+ html_tpl += chr(125) + ')' + chr(125) + ';'
542
+ html_tpl += '</script></body></html>'
543
+ return html_tpl
544
+
545
+ # POST request: JSON API prediction
546
+ try:
547
+ data = request.get_json()
548
+ if not data:
549
+ return jsonify({"error": "No JSON body. Send features as JSON.", "expected_features": list(RAW_FEATURE_INFO.keys())}), 400
550
+
551
+ # Check if caller wants DL mode, or if BEST_MODE is deep_learning, or if only DL is available
552
+ req_mode = data.pop('_mode', None)
553
+ use_dl = (req_mode == 'dl') or (req_mode != 'ml' and BEST_MODE == 'deep_learning' and DL_STATE is not None) or (MODEL_STATE is None and DL_STATE is not None)
554
+
555
+ if use_dl and DL_STATE:
556
+ # Deep Learning prediction
557
+ X = preprocess_dl_input(data)
558
+ dl_model = DL_STATE['model']
559
+ label_encoder = DL_STATE.get('label_encoder')
560
+ task_type = DL_STATE.get('task_type', 'classification')
561
+ classes = DL_STATE.get('classes', [])
562
+
563
+ pred = dl_model.predict(X)[0]
564
+ result = {"raw_prediction": int(pred) if isinstance(pred, (np.integer,)) else float(pred)}
565
+
566
+ if 'classif' in task_type.lower() and label_encoder:
567
+ try:
568
+ result['prediction'] = str(label_encoder.inverse_transform([int(pred)])[0])
569
+ except:
570
+ result['prediction'] = str(pred)
571
+ else:
572
+ result['prediction'] = float(pred)
573
+
574
+ if 'classif' in task_type.lower() and hasattr(dl_model, 'predict_proba'):
575
+ try:
576
+ proba = dl_model.predict_proba(X)[0]
577
+ result['confidence'] = float(max(proba))
578
+ if classes:
579
+ result['probabilities'] = {
580
+ str(c): round(float(p), 4) for c, p in zip(classes, proba)
581
+ }
582
+ except:
583
+ pass
584
+
585
+ result['mode'] = 'deep_learning'
586
+ return jsonify(result)
587
+
588
+ # Traditional/NLP prediction
589
+ model = MODEL_STATE['model']
590
+ target_encoder = MODEL_STATE.get('target_encoder')
591
+ task_type = MODEL_STATE.get('task_type', 'classification')
592
+
593
+ X = preprocess_input(data)
594
+
595
+ prediction = model.predict(X)[0]
596
+ result = {"raw_prediction": int(prediction) if isinstance(prediction, (np.integer,)) else float(prediction)}
597
+
598
+ if 'classif' in task_type.lower() and target_encoder:
599
+ try:
600
+ result['prediction'] = str(target_encoder.inverse_transform([int(prediction)])[0])
601
+ except:
602
+ result['prediction'] = str(prediction)
603
+ else:
604
+ result['prediction'] = float(prediction)
605
+
606
+ if 'classif' in task_type.lower() and hasattr(model, 'predict_proba'):
607
+ try:
608
+ proba = model.predict_proba(X)[0]
609
+ result['confidence'] = float(max(proba))
610
+ if target_encoder:
611
+ result['probabilities'] = {
612
+ str(c): round(float(p), 4) for c, p in zip(target_encoder.classes_, proba)
613
+ }
614
+ except:
615
+ pass
616
+
617
+ return jsonify(result)
618
+ except Exception as e:
619
+ return jsonify({"error": str(e)}), 500
620
+
621
+
622
+ if __name__ == '__main__':
623
+ parser = argparse.ArgumentParser()
624
+ parser.add_argument('--port', type=int, default=5000)
625
+ parser.add_argument('--host', type=str, default='0.0.0.0')
626
+ args = parser.parse_args()
627
+
628
+ if load_model():
629
+ print(f"\n🌐 API Server on http://{args.host}:{args.port}")
630
+ print(f" POST /predict — Send JSON with your feature values")
631
+ print(f" GET /health — Health check")
632
+ print(f" GET /model-info — Your model details and expected features")
633
+ app.run(host=args.host, port=args.port, debug=False)
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/config.json ADDED
@@ -0,0 +1,615 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "target_column": "price_range",
3
+ "feature_columns": [
4
+ "battery_power",
5
+ "blue",
6
+ "clock_speed",
7
+ "dual_sim",
8
+ "fc",
9
+ "four_g",
10
+ "int_memory",
11
+ "m_dep",
12
+ "mobile_wt",
13
+ "n_cores",
14
+ "pc",
15
+ "px_height",
16
+ "px_width",
17
+ "ram",
18
+ "sc_h",
19
+ "sc_w",
20
+ "talk_time",
21
+ "three_g",
22
+ "touch_screen",
23
+ "wifi"
24
+ ],
25
+ "task_type": "classification",
26
+ "best_mode": "traditional",
27
+ "modes_trained": [
28
+ "traditional"
29
+ ],
30
+ "best_model_name": "RandomForest",
31
+ "metrics": {
32
+ "accuracy": 0.8925,
33
+ "f1": 0.892925039212415,
34
+ "precision": 0.8937266322786124,
35
+ "recall": 0.8925,
36
+ "roc_auc": 0.9796333333333334
37
+ },
38
+ "data_summary": {
39
+ "rows": 2000,
40
+ "columns": 21,
41
+ "features_engineered": 20
42
+ },
43
+ "feature_metadata": [
44
+ {
45
+ "name": "battery_power",
46
+ "type": "numeric",
47
+ "min": 510.0,
48
+ "max": 1987.0,
49
+ "mean": 1238.513
50
+ },
51
+ {
52
+ "name": "blue",
53
+ "type": "numeric",
54
+ "min": 0.0,
55
+ "max": 1.0,
56
+ "mean": 0.495
57
+ },
58
+ {
59
+ "name": "clock_speed",
60
+ "type": "numeric",
61
+ "min": 0.5,
62
+ "max": 3.0,
63
+ "mean": 1.52225
64
+ },
65
+ {
66
+ "name": "dual_sim",
67
+ "type": "numeric",
68
+ "min": 0.0,
69
+ "max": 1.0,
70
+ "mean": 0.5095
71
+ },
72
+ {
73
+ "name": "fc",
74
+ "type": "numeric",
75
+ "min": 0.0,
76
+ "max": 16.0,
77
+ "mean": 4.294
78
+ },
79
+ {
80
+ "name": "four_g",
81
+ "type": "numeric",
82
+ "min": 0.0,
83
+ "max": 1.0,
84
+ "mean": 0.5215
85
+ },
86
+ {
87
+ "name": "int_memory",
88
+ "type": "numeric",
89
+ "min": 2.0,
90
+ "max": 64.0,
91
+ "mean": 32.0465
92
+ },
93
+ {
94
+ "name": "m_dep",
95
+ "type": "numeric",
96
+ "min": 0.1,
97
+ "max": 1.0,
98
+ "mean": 0.50175
99
+ },
100
+ {
101
+ "name": "mobile_wt",
102
+ "type": "numeric",
103
+ "min": 80.0,
104
+ "max": 199.0,
105
+ "mean": 140.2395
106
+ },
107
+ {
108
+ "name": "n_cores",
109
+ "type": "numeric",
110
+ "min": 1.0,
111
+ "max": 8.0,
112
+ "mean": 4.5205
113
+ },
114
+ {
115
+ "name": "pc",
116
+ "type": "numeric",
117
+ "min": 0.0,
118
+ "max": 20.0,
119
+ "mean": 9.9165
120
+ },
121
+ {
122
+ "name": "px_height",
123
+ "type": "numeric",
124
+ "min": 15.0,
125
+ "max": 1791.01,
126
+ "mean": 644.4581000000001
127
+ },
128
+ {
129
+ "name": "ram",
130
+ "type": "numeric",
131
+ "min": 296.99,
132
+ "max": 3958.01,
133
+ "mean": 2124.2565
134
+ },
135
+ {
136
+ "name": "sc_h",
137
+ "type": "numeric",
138
+ "min": 5.0,
139
+ "max": 19.0,
140
+ "mean": 12.3065
141
+ },
142
+ {
143
+ "name": "sc_w",
144
+ "type": "numeric",
145
+ "min": 0.0,
146
+ "max": 17.0,
147
+ "mean": 5.763
148
+ },
149
+ {
150
+ "name": "three_g",
151
+ "type": "numeric",
152
+ "min": 1.0,
153
+ "max": 1.0,
154
+ "mean": 1.0
155
+ },
156
+ {
157
+ "name": "touch_screen",
158
+ "type": "numeric",
159
+ "min": 0.0,
160
+ "max": 1.0,
161
+ "mean": 0.503
162
+ },
163
+ {
164
+ "name": "wifi",
165
+ "type": "numeric",
166
+ "min": 0.0,
167
+ "max": 1.0,
168
+ "mean": 0.507
169
+ },
170
+ {
171
+ "name": "talk_time",
172
+ "type": "datetime",
173
+ "min": "1970-01-01T00:00:00.000000002",
174
+ "max": "1970-01-01T00:00:00.000000020",
175
+ "placeholder": "Select date..."
176
+ }
177
+ ],
178
+ "numeric_features": [
179
+ {
180
+ "name": "battery_power",
181
+ "type": "numeric",
182
+ "min": 510.0,
183
+ "max": 1987.0,
184
+ "mean": 1238.513
185
+ },
186
+ {
187
+ "name": "blue",
188
+ "type": "numeric",
189
+ "min": 0.0,
190
+ "max": 1.0,
191
+ "mean": 0.495
192
+ },
193
+ {
194
+ "name": "clock_speed",
195
+ "type": "numeric",
196
+ "min": 0.5,
197
+ "max": 3.0,
198
+ "mean": 1.52225
199
+ },
200
+ {
201
+ "name": "dual_sim",
202
+ "type": "numeric",
203
+ "min": 0.0,
204
+ "max": 1.0,
205
+ "mean": 0.5095
206
+ },
207
+ {
208
+ "name": "fc",
209
+ "type": "numeric",
210
+ "min": 0.0,
211
+ "max": 16.0,
212
+ "mean": 4.294
213
+ },
214
+ {
215
+ "name": "four_g",
216
+ "type": "numeric",
217
+ "min": 0.0,
218
+ "max": 1.0,
219
+ "mean": 0.5215
220
+ },
221
+ {
222
+ "name": "int_memory",
223
+ "type": "numeric",
224
+ "min": 2.0,
225
+ "max": 64.0,
226
+ "mean": 32.0465
227
+ },
228
+ {
229
+ "name": "m_dep",
230
+ "type": "numeric",
231
+ "min": 0.1,
232
+ "max": 1.0,
233
+ "mean": 0.50175
234
+ },
235
+ {
236
+ "name": "mobile_wt",
237
+ "type": "numeric",
238
+ "min": 80.0,
239
+ "max": 199.0,
240
+ "mean": 140.2395
241
+ },
242
+ {
243
+ "name": "n_cores",
244
+ "type": "numeric",
245
+ "min": 1.0,
246
+ "max": 8.0,
247
+ "mean": 4.5205
248
+ },
249
+ {
250
+ "name": "pc",
251
+ "type": "numeric",
252
+ "min": 0.0,
253
+ "max": 20.0,
254
+ "mean": 9.9165
255
+ },
256
+ {
257
+ "name": "px_height",
258
+ "type": "numeric",
259
+ "min": 15.0,
260
+ "max": 1791.01,
261
+ "mean": 644.4581000000001
262
+ },
263
+ {
264
+ "name": "ram",
265
+ "type": "numeric",
266
+ "min": 296.99,
267
+ "max": 3958.01,
268
+ "mean": 2124.2565
269
+ },
270
+ {
271
+ "name": "sc_h",
272
+ "type": "numeric",
273
+ "min": 5.0,
274
+ "max": 19.0,
275
+ "mean": 12.3065
276
+ },
277
+ {
278
+ "name": "sc_w",
279
+ "type": "numeric",
280
+ "min": 0.0,
281
+ "max": 17.0,
282
+ "mean": 5.763
283
+ },
284
+ {
285
+ "name": "three_g",
286
+ "type": "numeric",
287
+ "min": 1.0,
288
+ "max": 1.0,
289
+ "mean": 1.0
290
+ },
291
+ {
292
+ "name": "touch_screen",
293
+ "type": "numeric",
294
+ "min": 0.0,
295
+ "max": 1.0,
296
+ "mean": 0.503
297
+ },
298
+ {
299
+ "name": "wifi",
300
+ "type": "numeric",
301
+ "min": 0.0,
302
+ "max": 1.0,
303
+ "mean": 0.507
304
+ }
305
+ ],
306
+ "categorical_features": [],
307
+ "text_features": [],
308
+ "nlp_text_column": null,
309
+ "results_per_mode": {
310
+ "traditional": {
311
+ "success": true,
312
+ "best_model": "RandomForest",
313
+ "metrics": {
314
+ "accuracy": 0.8925,
315
+ "f1": 0.892925039212415,
316
+ "precision": 0.8937266322786124,
317
+ "recall": 0.8925,
318
+ "roc_auc": 0.9796333333333334
319
+ },
320
+ "leaderboard": [
321
+ {
322
+ "name": "RandomForest",
323
+ "metrics": {
324
+ "accuracy": 0.8925,
325
+ "f1": 0.892925039212415,
326
+ "precision": 0.8937266322786124,
327
+ "recall": 0.8925,
328
+ "roc_auc": 0.9796333333333334
329
+ },
330
+ "score": 0.8925
331
+ }
332
+ ],
333
+ "task_type": "classification",
334
+ "algorithms_used": [
335
+ "random_forest"
336
+ ],
337
+ "feature_columns": [
338
+ "battery_power",
339
+ "blue",
340
+ "clock_speed",
341
+ "dual_sim",
342
+ "fc",
343
+ "four_g",
344
+ "int_memory",
345
+ "m_dep",
346
+ "mobile_wt",
347
+ "n_cores",
348
+ "pc",
349
+ "px_height",
350
+ "px_width",
351
+ "ram",
352
+ "sc_h",
353
+ "sc_w",
354
+ "talk_time",
355
+ "three_g",
356
+ "touch_screen",
357
+ "wifi"
358
+ ],
359
+ "feature_importance": [
360
+ {
361
+ "feature": "sc_h",
362
+ "importance": 0.5233,
363
+ "rank": 1
364
+ },
365
+ {
366
+ "feature": "battery_power",
367
+ "importance": 0.0759,
368
+ "rank": 2
369
+ },
370
+ {
371
+ "feature": "ram",
372
+ "importance": 0.0568,
373
+ "rank": 3
374
+ },
375
+ {
376
+ "feature": "px_height",
377
+ "importance": 0.0535,
378
+ "rank": 4
379
+ },
380
+ {
381
+ "feature": "mobile_wt",
382
+ "importance": 0.0384,
383
+ "rank": 5
384
+ },
385
+ {
386
+ "feature": "int_memory",
387
+ "importance": 0.0339,
388
+ "rank": 6
389
+ },
390
+ {
391
+ "feature": "touch_screen",
392
+ "importance": 0.0278,
393
+ "rank": 7
394
+ },
395
+ {
396
+ "feature": "pc",
397
+ "importance": 0.0271,
398
+ "rank": 8
399
+ },
400
+ {
401
+ "feature": "three_g",
402
+ "importance": 0.0262,
403
+ "rank": 9
404
+ },
405
+ {
406
+ "feature": "clock_speed",
407
+ "importance": 0.0243,
408
+ "rank": 10
409
+ },
410
+ {
411
+ "feature": "sc_w",
412
+ "importance": 0.0236,
413
+ "rank": 11
414
+ },
415
+ {
416
+ "feature": "m_dep",
417
+ "importance": 0.0226,
418
+ "rank": 12
419
+ },
420
+ {
421
+ "feature": "fc",
422
+ "importance": 0.022,
423
+ "rank": 13
424
+ },
425
+ {
426
+ "feature": "n_cores",
427
+ "importance": 0.0209,
428
+ "rank": 14
429
+ },
430
+ {
431
+ "feature": "four_g",
432
+ "importance": 0.0064,
433
+ "rank": 15
434
+ }
435
+ ],
436
+ "feature_metadata": [
437
+ {
438
+ "name": "battery_power",
439
+ "type": "numeric",
440
+ "min": 510.0,
441
+ "max": 1987.0,
442
+ "mean": 1238.513
443
+ },
444
+ {
445
+ "name": "blue",
446
+ "type": "numeric",
447
+ "min": 0.0,
448
+ "max": 1.0,
449
+ "mean": 0.495
450
+ },
451
+ {
452
+ "name": "clock_speed",
453
+ "type": "numeric",
454
+ "min": 0.5,
455
+ "max": 3.0,
456
+ "mean": 1.52225
457
+ },
458
+ {
459
+ "name": "dual_sim",
460
+ "type": "numeric",
461
+ "min": 0.0,
462
+ "max": 1.0,
463
+ "mean": 0.5095
464
+ },
465
+ {
466
+ "name": "fc",
467
+ "type": "numeric",
468
+ "min": 0.0,
469
+ "max": 16.0,
470
+ "mean": 4.294
471
+ },
472
+ {
473
+ "name": "four_g",
474
+ "type": "numeric",
475
+ "min": 0.0,
476
+ "max": 1.0,
477
+ "mean": 0.5215
478
+ },
479
+ {
480
+ "name": "int_memory",
481
+ "type": "numeric",
482
+ "min": 2.0,
483
+ "max": 64.0,
484
+ "mean": 32.0465
485
+ },
486
+ {
487
+ "name": "m_dep",
488
+ "type": "numeric",
489
+ "min": 0.1,
490
+ "max": 1.0,
491
+ "mean": 0.50175
492
+ },
493
+ {
494
+ "name": "mobile_wt",
495
+ "type": "numeric",
496
+ "min": 80.0,
497
+ "max": 199.0,
498
+ "mean": 140.2395
499
+ },
500
+ {
501
+ "name": "n_cores",
502
+ "type": "numeric",
503
+ "min": 1.0,
504
+ "max": 8.0,
505
+ "mean": 4.5205
506
+ },
507
+ {
508
+ "name": "pc",
509
+ "type": "numeric",
510
+ "min": 0.0,
511
+ "max": 20.0,
512
+ "mean": 9.9165
513
+ },
514
+ {
515
+ "name": "px_height",
516
+ "type": "numeric",
517
+ "min": 15.0,
518
+ "max": 1791.01,
519
+ "mean": 644.4581000000001
520
+ },
521
+ {
522
+ "name": "ram",
523
+ "type": "numeric",
524
+ "min": 296.99,
525
+ "max": 3958.01,
526
+ "mean": 2124.2565
527
+ },
528
+ {
529
+ "name": "sc_h",
530
+ "type": "numeric",
531
+ "min": 5.0,
532
+ "max": 19.0,
533
+ "mean": 12.3065
534
+ },
535
+ {
536
+ "name": "sc_w",
537
+ "type": "numeric",
538
+ "min": 0.0,
539
+ "max": 17.0,
540
+ "mean": 5.763
541
+ },
542
+ {
543
+ "name": "three_g",
544
+ "type": "numeric",
545
+ "min": 1.0,
546
+ "max": 1.0,
547
+ "mean": 1.0
548
+ },
549
+ {
550
+ "name": "touch_screen",
551
+ "type": "numeric",
552
+ "min": 0.0,
553
+ "max": 1.0,
554
+ "mean": 0.503
555
+ },
556
+ {
557
+ "name": "wifi",
558
+ "type": "numeric",
559
+ "min": 0.0,
560
+ "max": 1.0,
561
+ "mean": 0.507
562
+ },
563
+ {
564
+ "name": "talk_time",
565
+ "type": "datetime",
566
+ "min": "1970-01-01T00:00:00.000000002",
567
+ "max": "1970-01-01T00:00:00.000000020",
568
+ "placeholder": "Select date..."
569
+ }
570
+ ],
571
+ "n_rows": 2000,
572
+ "n_cols": 21,
573
+ "cleaned_file_path": "cleaned_1782413753.csv",
574
+ "reliability_score": 75,
575
+ "validation_warnings": null
576
+ }
577
+ },
578
+ "leaderboard": [
579
+ {
580
+ "mode": "Traditional ML",
581
+ "model": "RandomForest",
582
+ "score": 0.8925,
583
+ "metrics": {
584
+ "accuracy": 0.8925,
585
+ "f1": 0.892925039212415,
586
+ "precision": 0.8937266322786124,
587
+ "recall": 0.8925,
588
+ "roc_auc": 0.9796333333333334
589
+ }
590
+ }
591
+ ],
592
+ "actual_model_params": {
593
+ "bootstrap": true,
594
+ "ccp_alpha": 0.0,
595
+ "class_weight": "balanced",
596
+ "criterion": "gini",
597
+ "max_depth": 10,
598
+ "max_features": "sqrt",
599
+ "max_leaf_nodes": null,
600
+ "max_samples": null,
601
+ "min_impurity_decrease": 0.0,
602
+ "min_samples_leaf": 1,
603
+ "min_samples_split": 2,
604
+ "min_weight_fraction_leaf": 0.0,
605
+ "monotonic_cst": null,
606
+ "n_estimators": 150,
607
+ "n_jobs": -1,
608
+ "oob_score": false,
609
+ "random_state": 42,
610
+ "verbose": 0,
611
+ "warm_start": false
612
+ },
613
+ "actual_model_class": "RandomForestClassifier",
614
+ "has_production_engineer": true
615
+ }
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/evaluate.py ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 📊 Model Evaluation Script — Shows YOUR AutoML Results
3
+ ========================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ ╔════════════════════════════════════════════════════════════════════╗
7
+ ║ This script shows the EXACT SAME metrics as your AutoML training. ║
8
+ ║ Supports Traditional ML, Deep Learning, and NLP models. ║
9
+ ╚════════════════════════════════════════════════════════════════════╝
10
+
11
+ Model : RandomForest
12
+ Target Column : price_range
13
+ Task Type : classification
14
+
15
+ YOUR AUTOML METRICS:
16
+ # accuracy: 0.8925
17
+ # f1: 0.8929
18
+ # precision: 0.8937
19
+ # recall: 0.8925
20
+ # roc_auc: 0.9796
21
+
22
+ Usage:
23
+ python evaluate.py # Show YOUR AutoML results
24
+ python evaluate.py --reeval # Re-evaluate with data (may differ)
25
+ """
26
+
27
+ import os
28
+ import pickle
29
+ import argparse
30
+ import warnings
31
+ import numpy as np
32
+ import pandas as pd
33
+ from sklearn.model_selection import train_test_split, cross_val_score
34
+ from sklearn.preprocessing import LabelEncoder, StandardScaler
35
+ from sklearn.metrics import (
36
+ accuracy_score, precision_score, recall_score, f1_score,
37
+ r2_score, mean_absolute_error, mean_squared_error,
38
+ confusion_matrix, classification_report
39
+ )
40
+ try:
41
+ from sklearn.metrics import roc_auc_score
42
+ except ImportError:
43
+ roc_auc_score = None
44
+
45
+ warnings.filterwarnings('ignore')
46
+
47
+ TARGET_COLUMN = 'price_range'
48
+ TASK_TYPE = 'classification'
49
+ BEST_MODE = 'traditional' # Which mode won during AutoML
50
+
51
+
52
+ def print_classification_metrics(y_test, y_pred, model=None, X_test=None, target_encoder=None):
53
+ """Print all classification metrics including ROC-AUC."""
54
+ acc = accuracy_score(y_test, y_pred)
55
+ prec = precision_score(y_test, y_pred, average='weighted', zero_division=0)
56
+ rec = recall_score(y_test, y_pred, average='weighted', zero_division=0)
57
+ f1 = f1_score(y_test, y_pred, average='weighted', zero_division=0)
58
+ f1_macro = f1_score(y_test, y_pred, average='macro', zero_division=0)
59
+
60
+ print(f"\n✅ ACCURACY: {acc:.4f} ({acc*100:.1f}%)")
61
+ print(f" Precision: {prec:.4f} ({prec*100:.1f}%)")
62
+ print(f" Recall: {rec:.4f} ({rec*100:.1f}%)")
63
+ print(f" F1-Weighted: {f1:.4f} ({f1*100:.1f}%)")
64
+ print(f" F1-Macro: {f1_macro:.4f} ({f1_macro*100:.1f}%)")
65
+
66
+ # ROC-AUC
67
+ if roc_auc_score is not None and model is not None and X_test is not None:
68
+ try:
69
+ n_classes = len(np.unique(y_test))
70
+ if n_classes == 2 and hasattr(model, 'predict_proba'):
71
+ y_proba = model.predict_proba(X_test)[:, 1]
72
+ roc = roc_auc_score(y_test, y_proba)
73
+ print(f" ROC-AUC: {roc:.4f} ({roc*100:.1f}%)")
74
+ elif n_classes > 2 and hasattr(model, 'predict_proba'):
75
+ y_proba = model.predict_proba(X_test)
76
+ roc = roc_auc_score(y_test, y_proba, multi_class='ovr', average='weighted')
77
+ print(f" ROC-AUC: {roc:.4f} ({roc*100:.1f}%)")
78
+ elif n_classes == 2 and hasattr(model, 'decision_function'):
79
+ y_scores = model.decision_function(X_test)
80
+ roc = roc_auc_score(y_test, y_scores)
81
+ print(f" ROC-AUC: {roc:.4f} ({roc*100:.1f}%)")
82
+ except Exception:
83
+ pass
84
+
85
+ print("\n📊 Classification Report:")
86
+ if target_encoder is not None:
87
+ try:
88
+ print(classification_report(y_test, y_pred, target_names=target_encoder.classes_))
89
+ except Exception:
90
+ print(classification_report(y_test, y_pred))
91
+ else:
92
+ print(classification_report(y_test, y_pred))
93
+
94
+ print("\n📊 Confusion Matrix:")
95
+ cm = confusion_matrix(y_test, y_pred)
96
+ print(cm)
97
+
98
+
99
+ def print_regression_metrics(y_test, y_pred):
100
+ """Print all regression metrics."""
101
+ r2 = r2_score(y_test, y_pred)
102
+ mae = mean_absolute_error(y_test, y_pred)
103
+ mse = mean_squared_error(y_test, y_pred)
104
+ rmse = np.sqrt(mse)
105
+
106
+ print(f"\n✅ R² SCORE: {r2:.4f}")
107
+ print(f" MSE: {mse:.4f}")
108
+ print(f" RMSE: {rmse:.4f}")
109
+ print(f" MAE: {mae:.4f}")
110
+
111
+
112
+ def load_dl_model(dl_path='deep_learning_model.pkl'):
113
+ """Load the Deep Learning model."""
114
+ if not os.path.exists(dl_path):
115
+ return None
116
+ try:
117
+ with open(dl_path, 'rb') as f:
118
+ state = pickle.load(f)
119
+ return state
120
+ except Exception as e:
121
+ print(f"⚠️ Failed to load DL model: {e}")
122
+ return None
123
+
124
+
125
+ def main():
126
+ parser = argparse.ArgumentParser(description="Show your AutoML training results")
127
+ parser.add_argument('--model', type=str, default='model.pkl')
128
+ parser.add_argument('--dl-model', type=str, default='deep_learning_model.pkl', help='Path to DL model file')
129
+ parser.add_argument('--mode', type=str, default='auto', choices=['auto', 'traditional', 'dl'], help='Evaluation mode')
130
+ parser.add_argument('--data', type=str, default='data/cleaned_data.csv')
131
+ parser.add_argument('--reeval', action='store_true', help='Re-evaluate with fresh train/test split (may differ from AutoML)')
132
+ parser.add_argument('--cv', type=int, default=5, help='Number of cross-validation folds for --reeval')
133
+ args = parser.parse_args()
134
+
135
+ print("\n" + "=" * 60)
136
+ print("📊 DataVision AutoML — Your Training Results")
137
+ print("=" * 60)
138
+
139
+ # Decide which model to load based on BEST_MODE
140
+ use_dl = False
141
+ state = None
142
+
143
+ if args.mode == 'dl' or (args.mode == 'auto' and BEST_MODE == 'deep_learning'):
144
+ state = load_dl_model(args.dl_model)
145
+ if state:
146
+ use_dl = True
147
+ else:
148
+ print("⚠️ DL model not found, falling back to traditional ML model...")
149
+
150
+ if not use_dl and args.mode != 'dl':
151
+ if os.path.exists(args.model):
152
+ with open(args.model, 'rb') as f:
153
+ state = pickle.load(f)
154
+ elif os.path.exists(args.dl_model):
155
+ state = load_dl_model(args.dl_model)
156
+ if state:
157
+ use_dl = True
158
+
159
+ if state is None:
160
+ print("❌ No model found. Provide model.pkl or deep_learning_model.pkl")
161
+ return
162
+
163
+ model = state['model']
164
+ task_type = state.get('task_type', TASK_TYPE)
165
+ is_clf = 'classif' in task_type.lower()
166
+ mode_display = 'deep_learning' if use_dl else state.get('mode', 'traditional')
167
+ model_name = state.get('model_name', state.get('algorithm', 'Unknown'))
168
+
169
+ print(f"\n🏆 Model: {model_name}")
170
+ print(f"🎯 Task: {task_type}")
171
+ print(f"📊 Mode: {mode_display}")
172
+ if use_dl:
173
+ print(f"🧠 Architecture: {state.get('algorithm', 'MLP')}")
174
+ print(f" Features: {len(state.get('feature_columns', state.get('feature_metadata', [])))}")
175
+
176
+ # =========================================================================
177
+ # OPTION 1: Use SAVED y_test/y_pred (EXACT same as AutoML showed)
178
+ # =========================================================================
179
+ y_test_saved = state.get('y_test')
180
+ y_pred_saved = state.get('y_pred')
181
+
182
+ if not args.reeval and y_test_saved is not None and y_pred_saved is not None:
183
+ print("\n" + "=" * 60)
184
+ print("📋 YOUR AUTOML TRAINING RESULTS (from saved predictions)")
185
+ print("=" * 60)
186
+
187
+ y_test = np.array(y_test_saved)
188
+ y_pred = np.array(y_pred_saved)
189
+
190
+ # For ROC-AUC we need X_test and the model — try to reconstruct
191
+ X_test_for_roc = None
192
+ if use_dl:
193
+ X_test_saved = state.get('X_test')
194
+ if X_test_saved is not None:
195
+ X_test_for_roc = np.array(X_test_saved)
196
+
197
+ if is_clf:
198
+ print_classification_metrics(
199
+ y_test, y_pred, model=model,
200
+ X_test=X_test_for_roc,
201
+ target_encoder=state.get('target_encoder') if not use_dl else state.get('label_encoder')
202
+ )
203
+ else:
204
+ print_regression_metrics(y_test, y_pred)
205
+
206
+ # Also show stored metrics
207
+ stored_metrics = state.get('metrics', {})
208
+ if stored_metrics:
209
+ print("\n📋 Stored Metrics from Training:")
210
+ for k, v in stored_metrics.items():
211
+ if isinstance(v, float):
212
+ print(f" {k}: {v:.4f}")
213
+ else:
214
+ print(f" {k}: {v}")
215
+
216
+ print("\n" + "=" * 60)
217
+ print("✅ These are your EXACT AutoML results.")
218
+ print(" Use --reeval to re-evaluate with fresh data split (may differ).")
219
+ print("=" * 60)
220
+ return
221
+
222
+ # =========================================================================
223
+ # OPTION 2: Re-evaluate (may produce different results due to random split)
224
+ # =========================================================================
225
+ print("\n⚠️ Re-evaluating with fresh train/test split...")
226
+ print(" Note: Results may differ from AutoML due to different random split.")
227
+
228
+ if not os.path.exists(args.data):
229
+ print(f"⚠️ Data file not found: {args.data}")
230
+ print(" Showing stored metrics only:")
231
+ for k, v in state.get('metrics', {}).items():
232
+ print(f" {k}: {v}")
233
+ return
234
+
235
+ df = pd.read_csv(args.data)
236
+ target_col = state.get('target_column', TARGET_COLUMN)
237
+ feature_cols = state.get('feature_columns', [])
238
+
239
+ if target_col not in df.columns:
240
+ print(f"❌ Target '{target_col}' not in data")
241
+ return
242
+
243
+ df = df.dropna(subset=[target_col])
244
+
245
+ if use_dl:
246
+ # =====================================================================
247
+ # DL Re-evaluation: Use the DL model's preprocessing pipeline
248
+ # =====================================================================
249
+ feature_metadata = state.get('feature_metadata', [])
250
+ scaler = state.get('scaler')
251
+ label_encoder = state.get('label_encoder')
252
+
253
+ # Build feature matrix from metadata
254
+ X_parts = []
255
+ for meta in feature_metadata:
256
+ col = meta['name']
257
+ if meta.get('type') == 'numeric':
258
+ if col in df.columns:
259
+ X_parts.append(df[col].apply(pd.to_numeric, errors='coerce').fillna(meta.get('mean', 0)).values.reshape(-1, 1))
260
+ else:
261
+ X_parts.append(np.full((len(df), 1), meta.get('mean', 0)))
262
+ elif meta.get('type') == 'categorical':
263
+ if col in df.columns:
264
+ encoding = meta.get('encoding', {})
265
+ X_parts.append(df[col].astype(str).map(lambda v: encoding.get(v, 0)).values.reshape(-1, 1))
266
+ else:
267
+ X_parts.append(np.zeros((len(df), 1)))
268
+
269
+ if X_parts:
270
+ X = np.hstack(X_parts).astype(float)
271
+ else:
272
+ X = df.drop(columns=[target_col]).select_dtypes(include=[np.number]).fillna(0).values
273
+
274
+ y = df[target_col].values
275
+ if is_clf and label_encoder:
276
+ try:
277
+ y = label_encoder.transform(y)
278
+ except Exception:
279
+ le = LabelEncoder()
280
+ y = le.fit_transform(y)
281
+
282
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
283
+
284
+ # Scale AFTER split (same as training)
285
+ if scaler:
286
+ X_train = scaler.transform(X_train)
287
+ X_test = scaler.transform(X_test)
288
+
289
+ y_pred = model.predict(X_test)
290
+
291
+ if is_clf:
292
+ print(f"\n📊 Deep Learning Classification Metrics (re-evaluated):")
293
+ print_classification_metrics(y_test, y_pred, model=model, X_test=X_test,
294
+ target_encoder=label_encoder)
295
+ else:
296
+ print(f"\n📊 Deep Learning Regression Metrics (re-evaluated):")
297
+ print_regression_metrics(y_test, y_pred)
298
+ else:
299
+ # =====================================================================
300
+ # Traditional ML Re-evaluation
301
+ # =====================================================================
302
+ X = df.drop(columns=[target_col])
303
+ y = df[target_col]
304
+
305
+ target_encoder = state.get('target_encoder')
306
+ if is_clf and target_encoder:
307
+ y = pd.Series(target_encoder.transform(y))
308
+
309
+ label_encoders = state.get('label_encoders', {})
310
+ for col in X.select_dtypes(include=['object', 'category']).columns:
311
+ if col in label_encoders:
312
+ try:
313
+ X[col] = label_encoders[col].transform(X[col].astype(str))
314
+ except:
315
+ le = LabelEncoder()
316
+ X[col] = le.fit_transform(X[col].astype(str))
317
+ else:
318
+ le = LabelEncoder()
319
+ X[col] = le.fit_transform(X[col].astype(str))
320
+
321
+ X = X.fillna(0)
322
+
323
+ scaler = state.get('scaler')
324
+ if scaler and feature_cols:
325
+ avail = [c for c in feature_cols if c in X.columns]
326
+ if avail:
327
+ X = X[avail]
328
+ X = pd.DataFrame(scaler.transform(X), columns=avail)
329
+
330
+ # Align columns with model's expected features
331
+ if feature_cols:
332
+ for col in feature_cols:
333
+ if col not in X.columns:
334
+ X[col] = 0
335
+ X = X.reindex(columns=feature_cols, fill_value=0)
336
+
337
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
338
+
339
+ y_pred = model.predict(X_test)
340
+ y_pred_train = model.predict(X_train)
341
+
342
+ if is_clf:
343
+ print(f"\n📊 Classification Metrics (re-evaluated):")
344
+ print_classification_metrics(y_test, y_pred, model=model, X_test=X_test,
345
+ target_encoder=state.get('target_encoder'))
346
+
347
+ # Overfitting check
348
+ train_acc = accuracy_score(y_train, y_pred_train)
349
+ test_acc = accuracy_score(y_test, y_pred)
350
+ gap = train_acc - test_acc
351
+ if gap > 0.1:
352
+ print(f"\n⚠️ Overfitting detected (train-test gap: {gap:.4f})")
353
+ elif gap < 0.01:
354
+ print(f"\n✅ Good generalization (train-test gap: {gap:.4f})")
355
+
356
+ try:
357
+ cv_scores = cross_val_score(model, X, y, cv=args.cv, scoring='accuracy')
358
+ print(f"\n🔄 {args.cv}-Fold CV Accuracy: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")
359
+ except Exception as e:
360
+ print(f" CV failed: {e}")
361
+ else:
362
+ print(f"\n📊 Regression Metrics (re-evaluated):")
363
+ print_regression_metrics(y_test, y_pred)
364
+
365
+ try:
366
+ cv_scores = cross_val_score(model, X, y, cv=args.cv, scoring='r2')
367
+ print(f"\n🔄 {args.cv}-Fold CV R²: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")
368
+ except:
369
+ pass
370
+
371
+ print("\n✅ Evaluation complete!")
372
+
373
+
374
+ if __name__ == '__main__':
375
+ main()
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/predict.py ADDED
@@ -0,0 +1,728 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🔮 Prediction Script — Uses Your ACTUAL Trained Model
3
+ ========================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ ╔════════════════════════════════════════════════════════════════════╗
7
+ ║ This script loads YOUR model.pkl that was trained by AutoML. ║
8
+ ║ The predictions match EXACTLY what you saw in DataVision. ║
9
+ ╚════════════════════════════════════════════════════════════════════╝
10
+
11
+ YOUR MODEL:
12
+ 🏆 Model : RandomForest
13
+ 🎯 Target : price_range
14
+ 📊 Task : classification
15
+ 📁 Features : 20 columns
16
+
17
+ Usage:
18
+ python predict.py # Interactive mode
19
+ python predict.py --json '{"col1": 1, "col2": "A"}' # JSON input
20
+ python predict.py --csv input.csv # Batch predictions
21
+ """
22
+
23
+ import os
24
+ import pickle
25
+ import argparse
26
+ import json
27
+ import re
28
+ import warnings
29
+ import numpy as np
30
+ import pandas as pd
31
+ from sklearn.preprocessing import LabelEncoder, RobustScaler
32
+
33
+ warnings.filterwarnings('ignore')
34
+
35
+ # Your actual feature columns from training
36
+ FEATURE_COLUMNS = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt', 'n_cores', 'pc', 'px_height', 'px_width', 'ram', 'sc_h', 'sc_w', 'talk_time', 'three_g', 'touch_screen', 'wifi']
37
+ TARGET_COLUMN = 'price_range'
38
+ BEST_MODE = 'traditional' # Which mode won during AutoML
39
+
40
+ # Feature metadata with actual types and ranges from your dataset
41
+ FEATURE_INFO = {
42
+ "battery_power": {"type": "numeric", "min": 510.0, "max": 1987.0, "default": 1238.51},
43
+ "blue": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.49},
44
+ "clock_speed": {"type": "numeric", "min": 0.5, "max": 3.0, "default": 1.52},
45
+ "dual_sim": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
46
+ "fc": {"type": "numeric", "min": 0.0, "max": 16.0, "default": 4.29},
47
+ "four_g": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.52},
48
+ "int_memory": {"type": "numeric", "min": 2.0, "max": 64.0, "default": 32.05},
49
+ "m_dep": {"type": "numeric", "min": 0.1, "max": 1.0, "default": 0.50},
50
+ "mobile_wt": {"type": "numeric", "min": 80.0, "max": 199.0, "default": 140.24},
51
+ "n_cores": {"type": "numeric", "min": 1.0, "max": 8.0, "default": 4.52},
52
+ "pc": {"type": "numeric", "min": 0.0, "max": 20.0, "default": 9.92},
53
+ "px_height": {"type": "numeric", "min": 15.0, "max": 1791.01, "default": 644.46},
54
+ "ram": {"type": "numeric", "min": 296.99, "max": 3958.01, "default": 2124.26},
55
+ "sc_h": {"type": "numeric", "min": 5.0, "max": 19.0, "default": 12.31},
56
+ "sc_w": {"type": "numeric", "min": 0.0, "max": 17.0, "default": 5.76},
57
+ "three_g": {"type": "numeric", "min": 1.0, "max": 1.0, "default": 1.00},
58
+ "touch_screen": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.50},
59
+ "wifi": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
60
+ }
61
+
62
+
63
+ # =============================================================================
64
+ # Standalone Feature Engineer — Replicates DataVision's EXACT preprocessing
65
+ # =============================================================================
66
+ class StandaloneFeatureEngineer:
67
+ """Replicate DataVision's ProductionFeatureEngineer.transform() pipeline.
68
+
69
+ Loaded from feature_engineer.pkl which contains all fitted sklearn
70
+ transformers, encoders, and feature names from your AutoML training.
71
+ """
72
+
73
+ def __init__(self, state):
74
+ self.transformers = state.get('transformers', {})
75
+ self.encoders = state.get('encoders', {})
76
+ self.feature_names = state.get('feature_names', [])
77
+ self.original_columns = state.get('original_columns', [])
78
+ self.selected_feature_indices = state.get('selected_feature_indices', None)
79
+ self.is_fitted = state.get('is_fitted', True)
80
+ # Stopwords / sentiment words for text features
81
+ self.stopwords = set("i me my myself we our ours ourselves you your yours yourself yourselves he him his himself she her hers herself it its itself they them their theirs themselves what which who whom this that these those am is are was were be been being have has had having do does did doing a an the and but if or because as until while of at by for with about against between through during before after above below to from up down in out on off over under again further then once here there when where why how all both each few more most other some such no nor not only own same so than too very s t can will just don should now d ll m o re ve y ain aren couldn didn doesn hadn hasn haven isn ma mightn mustn needn shan shouldn wasn weren won wouldn".split())
82
+ self.positive_words = set("good great excellent amazing wonderful fantastic brilliant outstanding superb love best perfect beautiful awesome incredible impressive magnificent exceptional fabulous terrific marvelous stellar remarkable extraordinary delightful gorgeous splendid spectacular phenomenal exquisite lovely charming pleasant nice fine happy joy enjoy satisfied pleased glad cheerful excited thrilled".split())
83
+ self.negative_words = set("bad terrible horrible awful worst poor disappointing disgusting pathetic useless dreadful atrocious appalling abysmal mediocre inferior unpleasant nasty vile wretched miserable lousy horrendous hideous ghastly hideous deplorable lamentable pitiful woeful tragic sad angry frustrated annoyed irritated disappointed upset unhappy distressed sorrowful gloomy depressed".split())
84
+
85
+ def _clean_text(self, text):
86
+ text = str(text).lower()
87
+ text = re.sub(r'<[^>]+>', ' ', text)
88
+ text = re.sub(r'http\S+', ' ', text)
89
+ text = re.sub(r'[^a-z0-9\s]', ' ', text)
90
+ text = ' '.join(w for w in text.split() if w not in self.stopwords and len(w) > 1)
91
+ return text if text.strip() else 'empty'
92
+
93
+ def _text_statistics(self, text):
94
+ text = str(text)
95
+ words = text.split()
96
+ word_lengths = [len(w) for w in words] if words else [0]
97
+ unique_ratio = len(set(words)) / max(len(words), 1)
98
+ features = [
99
+ len(text), len(words), np.mean(word_lengths), np.std(word_lengths),
100
+ np.max(word_lengths) if word_lengths else 0, unique_ratio,
101
+ text.count('!'), text.count('?'),
102
+ sum(1 for c in text if c.isupper()) / max(len(text), 1),
103
+ len([w for w in words if len(w) > 6]) / max(len(words), 1),
104
+ ]
105
+ return features
106
+
107
+ def _sentiment_score(self, text):
108
+ words = set(str(text).lower().split())
109
+ pos = len(words & self.positive_words)
110
+ neg = len(words & self.negative_words)
111
+ total = max(pos + neg, 1)
112
+ return [pos / total, neg / total, (pos - neg) / total, total]
113
+
114
+ def transform(self, df):
115
+ if not self.is_fitted:
116
+ raise ValueError("Feature engineer not fitted!")
117
+ df = df.copy()
118
+
119
+ # Handle duplicate column names
120
+ cols = df.columns.tolist()
121
+ seen = {}
122
+ new_cols = []
123
+ for col in cols:
124
+ if col in seen:
125
+ seen[col] += 1
126
+ new_cols.append(f"{col}_{seen[col]}")
127
+ else:
128
+ seen[col] = 0
129
+ new_cols.append(col)
130
+ df.columns = new_cols
131
+
132
+ feature_parts = []
133
+
134
+ # ===== NUMERIC FEATURES =====
135
+ if 'numeric_cols' in self.transformers:
136
+ numeric_cols = self.transformers['numeric_cols']
137
+ for col in numeric_cols:
138
+ if col not in df.columns:
139
+ df[col] = 0
140
+ try:
141
+ numeric_data = df[numeric_cols].apply(pd.to_numeric, errors='coerce').fillna(0).values
142
+ scaler = self.transformers['numeric_scaler']
143
+ numeric_scaled = scaler.transform(numeric_data)
144
+ numeric_scaled = np.nan_to_num(numeric_scaled, nan=0.0, posinf=0.0, neginf=0.0)
145
+ feature_parts.append(numeric_scaled)
146
+
147
+ # Polynomial features
148
+ if 2 <= len(numeric_cols) <= 10:
149
+ interactions = []
150
+ for i in range(min(len(numeric_cols), 4)):
151
+ interactions.append((numeric_scaled[:, i] ** 2).reshape(-1, 1))
152
+ for i in range(min(len(numeric_cols), 4)):
153
+ for j in range(i + 1, min(len(numeric_cols), 4)):
154
+ interactions.append((numeric_scaled[:, i] * numeric_scaled[:, j]).reshape(-1, 1))
155
+ if interactions:
156
+ feature_parts.append(np.hstack(interactions))
157
+ except Exception as e:
158
+ print(f" Warning: Numeric transform error: {e}")
159
+ feature_parts.append(np.zeros((len(df), len(numeric_cols))))
160
+
161
+ # ===== CATEGORICAL: One-hot =====
162
+ for key, cols in self.transformers.items():
163
+ if key.endswith('_onehot'):
164
+ col_name = key.replace('_onehot', '')
165
+ try:
166
+ if col_name in df.columns:
167
+ series = df[col_name].fillna('_MISSING_').astype(str).str.strip()
168
+ dummies = pd.get_dummies(series, prefix=col_name)
169
+ dummies = dummies.reindex(columns=cols, fill_value=0)
170
+ feature_parts.append(dummies.values.astype(float))
171
+ else:
172
+ feature_parts.append(np.zeros((len(df), len(cols))))
173
+ except:
174
+ feature_parts.append(np.zeros((len(df), len(cols))))
175
+
176
+ # ===== CATEGORICAL: Label + Frequency =====
177
+ for col, le in self.encoders.items():
178
+ try:
179
+ if col in df.columns:
180
+ series = df[col].fillna('_MISSING_').astype(str).str.strip()
181
+ encoded = series.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
182
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
183
+ freq_key = f'{col}_freq_map'
184
+ if freq_key in self.transformers:
185
+ freq_map = self.transformers[freq_key]
186
+ freq_encoded = series.map(lambda s: freq_map.get(s, 0.0)).values.astype(float)
187
+ feature_parts.append(freq_encoded.reshape(-1, 1))
188
+ else:
189
+ feature_parts.append(np.zeros((len(df), 1)))
190
+ if f'{col}_freq_map' in self.transformers:
191
+ feature_parts.append(np.zeros((len(df), 1)))
192
+ except:
193
+ feature_parts.append(np.zeros((len(df), 1)))
194
+
195
+ # ===== CATEGORICAL INTERACTIONS =====
196
+ if 'cat_interactions' in self.transformers:
197
+ interaction_encoders = self.transformers['cat_interactions']
198
+ for key, value in interaction_encoders.items():
199
+ if key.endswith('_freq_map'):
200
+ continue
201
+ try:
202
+ col1, col2, le = value
203
+ if col1 in df.columns and col2 in df.columns:
204
+ combined = df[col1].fillna('_NA_').astype(str) + "_X_" + df[col2].fillna('_NA_').astype(str)
205
+ encoded = combined.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
206
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
207
+ freq_key = f"{key}_freq_map"
208
+ if freq_key in interaction_encoders:
209
+ freq_map = interaction_encoders[freq_key]
210
+ freq_encoded = combined.map(lambda s: freq_map.get(s, 0.0)).values.astype(float)
211
+ feature_parts.append(freq_encoded.reshape(-1, 1))
212
+ else:
213
+ feature_parts.append(np.zeros((len(df), 1)))
214
+ if f"{key}_freq_map" in interaction_encoders:
215
+ feature_parts.append(np.zeros((len(df), 1)))
216
+ except:
217
+ pass
218
+
219
+ # ===== NUMERIC-CATEGORICAL AGGREGATES =====
220
+ if 'num_cat_aggs' in self.transformers:
221
+ agg_maps = self.transformers['num_cat_aggs']
222
+ for key, mean_map in agg_maps.items():
223
+ try:
224
+ parts = key.rsplit('_by_', 1)
225
+ if len(parts) == 2:
226
+ num_col = parts[0]
227
+ cat_col = parts[1].rsplit('_mean', 1)[0]
228
+ if num_col in df.columns and cat_col in df.columns:
229
+ cat_series = df[cat_col].fillna('_NA_').astype(str)
230
+ default_val = np.mean(list(mean_map.values())) if mean_map else 0
231
+ agg_values = cat_series.map(lambda s: mean_map.get(s, default_val)).values.astype(float)
232
+ feature_parts.append(agg_values.reshape(-1, 1))
233
+ deviation = df[num_col].fillna(0).values - agg_values
234
+ feature_parts.append(deviation.reshape(-1, 1))
235
+ else:
236
+ feature_parts.append(np.zeros((len(df), 1)))
237
+ feature_parts.append(np.zeros((len(df), 1)))
238
+ except:
239
+ pass
240
+
241
+ # ===== TEXT / NLP FEATURES =====
242
+ for key, tfidf in self.transformers.items():
243
+ if key.endswith('_tfidf'):
244
+ col = key.replace('_tfidf', '')
245
+ try:
246
+ if col in df.columns:
247
+ series = df[col].fillna('').astype(str)
248
+ cleaned = series.apply(self._clean_text)
249
+ tfidf_matrix = tfidf.transform(cleaned)
250
+ svd_key = f'{col}_svd'
251
+ if svd_key in self.transformers:
252
+ text_features = self.transformers[svd_key].transform(tfidf_matrix)
253
+ else:
254
+ text_features = tfidf_matrix.toarray()
255
+ feature_parts.append(text_features)
256
+ text_stats = np.array([self._text_statistics(t) for t in series])
257
+ feature_parts.append(text_stats)
258
+ sentiment = np.array([self._sentiment_score(t) for t in series])
259
+ feature_parts.append(sentiment)
260
+ except:
261
+ pass
262
+
263
+ # ===== COMBINE =====
264
+ if not feature_parts:
265
+ return np.zeros((len(df), len(self.feature_names)))
266
+
267
+ X = np.hstack(feature_parts)
268
+ X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
269
+
270
+ # Variance selector
271
+ if 'variance_selector' in self.transformers:
272
+ selector = self.transformers['variance_selector']
273
+ try:
274
+ if X.shape[1] == selector.n_features_in_:
275
+ X = selector.transform(X)
276
+ elif hasattr(self, 'selected_feature_indices') and self.selected_feature_indices is not None:
277
+ valid_idx = [i for i in self.selected_feature_indices if i < X.shape[1]]
278
+ if valid_idx:
279
+ X = X[:, valid_idx]
280
+ except:
281
+ pass
282
+
283
+ # PCA (if used during training)
284
+ if 'pca' in self.transformers:
285
+ try:
286
+ pca = self.transformers['pca']
287
+ if X.shape[1] == pca.n_features_in_:
288
+ X = pca.transform(X)
289
+ except:
290
+ pass
291
+
292
+ # Pad/truncate to match training dimensions
293
+ if X.shape[1] != len(self.feature_names):
294
+ if X.shape[1] < len(self.feature_names):
295
+ padding = np.zeros((len(df), len(self.feature_names) - X.shape[1]))
296
+ X = np.hstack([X, padding])
297
+ else:
298
+ X = X[:, :len(self.feature_names)]
299
+
300
+ return X
301
+
302
+ def transform_single(self, data):
303
+ for col in self.original_columns:
304
+ if col not in data:
305
+ data[col] = 0
306
+ df = pd.DataFrame([data])
307
+ X = self.transform(df)
308
+ expected = len(self.feature_names)
309
+ if X.shape[1] != expected:
310
+ if X.shape[1] < expected:
311
+ X = np.hstack([X, np.zeros((1, expected - X.shape[1]))])
312
+ else:
313
+ X = X[:, :expected]
314
+ return X
315
+
316
+
317
+ # =============================================================================
318
+ # Load model and feature engineer
319
+ # =============================================================================
320
+ FEATURE_ENGINEER = None
321
+
322
+ def _load_feature_engineer():
323
+ global FEATURE_ENGINEER
324
+ if FEATURE_ENGINEER is not None:
325
+ return FEATURE_ENGINEER
326
+ eng_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'feature_engineer.pkl')
327
+ if os.path.exists(eng_path):
328
+ try:
329
+ with open(eng_path, 'rb') as f:
330
+ eng_state = pickle.load(f)
331
+ FEATURE_ENGINEER = StandaloneFeatureEngineer(eng_state)
332
+ print(f" Pipeline: feature_engineer.pkl ({len(FEATURE_ENGINEER.feature_names)} engineered features)")
333
+ return FEATURE_ENGINEER
334
+ except Exception as e:
335
+ print(f" Warning: Could not load feature_engineer.pkl: {e}")
336
+ return None
337
+
338
+
339
+ def load_model(model_path='model.pkl'):
340
+ """Load the trained model and its components."""
341
+ if not os.path.exists(model_path):
342
+ print(f"❌ Model not found: {model_path}")
343
+ print(" Run the training script first, or ensure model.pkl is in this directory.")
344
+ return None
345
+
346
+ with open(model_path, 'rb') as f:
347
+ state = pickle.load(f)
348
+
349
+ print(f"✅ Loaded model: {state.get('model_name', 'Unknown')}")
350
+ print(f" Task: {state.get('task_type', 'Unknown')}")
351
+ print(f" Target: {state.get('target_column', 'Unknown')}")
352
+
353
+ # Load feature engineer (exact preprocessing pipeline)
354
+ _load_feature_engineer()
355
+ return state
356
+
357
+
358
+ def predict_single(state, input_data: dict):
359
+ """Make a prediction for a single data point.
360
+
361
+ Uses the EXACT same preprocessing pipeline as DataVision AutoML:
362
+ 1. StandaloneFeatureEngineer from feature_engineer.pkl (preferred)
363
+ 2. Fallback to basic label-encode + scale (less accurate)
364
+ """
365
+ model = state['model']
366
+ feature_cols = state.get('feature_columns', FEATURE_COLUMNS)
367
+ target_encoder = state.get('target_encoder', None)
368
+ task_type = state.get('task_type', 'classification')
369
+
370
+ # Use feature engineer for EXACT AutoML-matching preprocessing
371
+ engineer = _load_feature_engineer()
372
+ if engineer is not None:
373
+ try:
374
+ X = engineer.transform_single(input_data)
375
+ except Exception as e:
376
+ print(f"Warning: Feature engineer failed ({e}), using fallback")
377
+ X = _fallback_preprocess(input_data, state)
378
+ else:
379
+ X = _fallback_preprocess(input_data, state)
380
+
381
+ prediction = model.predict(X)[0]
382
+ result = {'raw_prediction': prediction}
383
+
384
+ if 'classif' in task_type.lower() and target_encoder is not None:
385
+ try:
386
+ result['prediction'] = target_encoder.inverse_transform([int(prediction)])[0]
387
+ except:
388
+ result['prediction'] = prediction
389
+ else:
390
+ result['prediction'] = float(prediction)
391
+
392
+ if 'classif' in task_type.lower() and hasattr(model, 'predict_proba'):
393
+ try:
394
+ proba = model.predict_proba(X)[0]
395
+ result['confidence'] = float(max(proba))
396
+ if target_encoder is not None:
397
+ result['probabilities'] = {
398
+ str(cls): float(p) for cls, p in zip(target_encoder.classes_, proba)
399
+ }
400
+ else:
401
+ result['probabilities'] = {str(i): float(p) for i, p in enumerate(proba)}
402
+ except:
403
+ pass
404
+
405
+ return result
406
+
407
+
408
+ def _fallback_preprocess(input_data, state):
409
+ """Fallback: basic label-encode + scale (used only if feature_engineer.pkl missing)."""
410
+ feature_cols = state.get('feature_columns', FEATURE_COLUMNS)
411
+ label_encoders = state.get('label_encoders', {})
412
+ scaler = state.get('scaler', None)
413
+
414
+ row = {}
415
+ for col in feature_cols:
416
+ val = input_data.get(col, 0)
417
+ if col in label_encoders:
418
+ le = label_encoders[col]
419
+ val_str = str(val)
420
+ val = le.transform([val_str])[0] if val_str in le.classes_ else 0
421
+ else:
422
+ try:
423
+ val = float(val)
424
+ except (ValueError, TypeError):
425
+ val = 0
426
+ row[col] = val
427
+
428
+ X_proc = pd.DataFrame([row], columns=feature_cols)
429
+ if scaler is not None:
430
+ try:
431
+ X_proc = pd.DataFrame(scaler.transform(X_proc), columns=feature_cols)
432
+ except:
433
+ pass
434
+ return X_proc
435
+
436
+
437
+ def predict_batch(state, csv_path: str, output_path: str = None):
438
+ """Make predictions for a CSV file."""
439
+ df = pd.read_csv(csv_path)
440
+ print(f"\n📊 Predicting for {len(df)} rows...")
441
+
442
+ predictions = []
443
+ for _, row in df.iterrows():
444
+ result = predict_single(state, row.to_dict())
445
+ predictions.append(result['prediction'])
446
+
447
+ df['prediction'] = predictions
448
+ if output_path is None:
449
+ output_path = csv_path.replace('.csv', '_predictions.csv')
450
+ df.to_csv(output_path, index=False)
451
+ print(f"💾 Predictions saved to: {output_path}")
452
+ return df
453
+
454
+
455
+ # =============================================================================
456
+ # Deep Learning Model Support (sklearn MLPClassifier/MLPRegressor)
457
+ # =============================================================================
458
+ DL_STATE = None
459
+
460
+ def load_dl_model(dl_path='deep_learning_model.pkl'):
461
+ """Load the Deep Learning model (sklearn MLP) with its preprocessing state."""
462
+ global DL_STATE
463
+ if not os.path.exists(dl_path):
464
+ return None
465
+ try:
466
+ with open(dl_path, 'rb') as f:
467
+ DL_STATE = pickle.load(f)
468
+ print(f"✅ Loaded DL model: {DL_STATE.get('algorithm', 'MLP')}")
469
+ print(f" Task: {DL_STATE.get('task_type', 'Unknown')}")
470
+ print(f" Features: {len(DL_STATE.get('feature_metadata', []))}")
471
+ return DL_STATE
472
+ except Exception as e:
473
+ print(f"⚠️ Failed to load DL model: {e}")
474
+ return None
475
+
476
+
477
+ def predict_single_dl(dl_state, input_data: dict):
478
+ """Make a prediction using the Deep Learning model.
479
+
480
+ Uses feature_metadata for correct preprocessing:
481
+ - Numeric features: float conversion, fillna with mean
482
+ - Categorical features: one-hot encoding matching training
483
+ - StandardScaler applied to full feature vector
484
+ """
485
+ model = dl_state['model']
486
+ scaler = dl_state.get('scaler')
487
+ label_encoder = dl_state.get('label_encoder')
488
+ feature_metadata = dl_state.get('feature_metadata', [])
489
+ task_type = dl_state.get('task_type', 'classification')
490
+ classes = dl_state.get('classes', [])
491
+
492
+ # Build feature vector using feature_metadata (same as DL engine)
493
+ feature_values = []
494
+
495
+ # Process numeric features first
496
+ for meta in feature_metadata:
497
+ if meta.get('type') == 'numeric':
498
+ col = meta['name']
499
+ if col in input_data:
500
+ try:
501
+ feature_values.append(float(input_data[col]))
502
+ except (ValueError, TypeError):
503
+ feature_values.append(meta.get('mean', 0))
504
+ else:
505
+ feature_values.append(meta.get('mean', 0))
506
+
507
+ # Process categorical features (one-hot encoded during training)
508
+ for meta in feature_metadata:
509
+ if meta.get('type') == 'categorical':
510
+ col = meta['name']
511
+ value = str(input_data.get(col, ''))
512
+ options = meta.get('options', [])
513
+
514
+ # One-hot encode: one value per option
515
+ for opt in options:
516
+ feature_values.append(1.0 if value == opt else 0.0)
517
+ # NaN dummy (dummy_na=True during training)
518
+ feature_values.append(1.0 if not value or value == 'nan' else 0.0)
519
+
520
+ # Fallback to feature_columns if no metadata
521
+ if not feature_values and dl_state.get('feature_columns'):
522
+ for col in dl_state['feature_columns']:
523
+ if col in input_data:
524
+ try:
525
+ feature_values.append(float(input_data[col]) if not isinstance(input_data[col], str) else 0)
526
+ except:
527
+ feature_values.append(0)
528
+ else:
529
+ feature_values.append(0)
530
+
531
+ X = np.array([feature_values])
532
+
533
+ # Handle dimension mismatch with scaler
534
+ if scaler and hasattr(scaler, 'n_features_in_'):
535
+ expected = scaler.n_features_in_
536
+ actual = X.shape[1]
537
+ if actual < expected:
538
+ padding = np.zeros((1, expected - actual))
539
+ X = np.hstack([X, padding])
540
+ elif actual > expected:
541
+ X = X[:, :expected]
542
+
543
+ # Scale
544
+ if scaler:
545
+ X_scaled = scaler.transform(X)
546
+ else:
547
+ X_scaled = X
548
+
549
+ # Predict
550
+ pred = model.predict(X_scaled)[0]
551
+ result = {'raw_prediction': pred}
552
+
553
+ if 'classif' in task_type.lower() and label_encoder is not None:
554
+ try:
555
+ result['prediction'] = str(label_encoder.inverse_transform([int(pred)])[0])
556
+ except:
557
+ result['prediction'] = str(pred)
558
+ else:
559
+ result['prediction'] = float(pred)
560
+
561
+ if 'classif' in task_type.lower() and hasattr(model, 'predict_proba'):
562
+ try:
563
+ proba = model.predict_proba(X_scaled)[0]
564
+ result['confidence'] = float(max(proba))
565
+ if classes:
566
+ result['probabilities'] = {
567
+ str(cls): round(float(p), 4) for cls, p in zip(classes, proba)
568
+ }
569
+ except:
570
+ pass
571
+
572
+ return result
573
+
574
+
575
+ def interactive_mode(state, use_dl=False):
576
+ """Interactive prediction mode with actual feature names and types."""
577
+ print("\n" + "=" * 50)
578
+ print("🔮 Interactive Prediction Mode")
579
+ if use_dl:
580
+ print(" (Using Deep Learning model)")
581
+ print("=" * 50)
582
+ print("Enter values for each feature:\n")
583
+
584
+ # For DL mode, use feature_metadata from DL state
585
+ if use_dl and DL_STATE:
586
+ input_data = {}
587
+ for meta in DL_STATE.get('feature_metadata', []):
588
+ col = meta['name']
589
+ ftype = meta.get('type', 'numeric')
590
+ if ftype == 'numeric':
591
+ mn = meta.get('min', '?')
592
+ mx = meta.get('max', '?')
593
+ default = meta.get('mean', 0)
594
+ val = input(f" {col} [{mn} - {mx}] (default: {default:.2f}): ").strip()
595
+ input_data[col] = float(val) if val else default
596
+ elif ftype == 'categorical':
597
+ options = meta.get('options', [])
598
+ print(f" {col} options: {options}")
599
+ val = input(f" {col} (pick one): ").strip()
600
+ input_data[col] = val if val else (options[0] if options else '')
601
+ result = predict_single_dl(DL_STATE, input_data)
602
+ else:
603
+ input_data = {}
604
+ for col, info in FEATURE_INFO.items():
605
+ ftype = info.get('type', 'numeric')
606
+
607
+ if ftype == 'numeric':
608
+ mn = info.get('min', '?')
609
+ mx = info.get('max', '?')
610
+ default = info.get('default', 0)
611
+ val = input(f" {col} [{mn} - {mx}] (default: {default}): ").strip()
612
+ input_data[col] = float(val) if val else default
613
+
614
+ elif ftype == 'categorical':
615
+ options = info.get('options', [])
616
+ print(f" {col} options: {options}")
617
+ val = input(f" {col} (pick one): ").strip()
618
+ input_data[col] = val if val else (options[0] if options else '')
619
+
620
+ elif ftype == 'text':
621
+ val = input(f" {col} (enter text): ").strip()
622
+ input_data[col] = val if val else ''
623
+ result = predict_single(state, input_data)
624
+
625
+ print(f"\n🎯 Prediction: {result['prediction']}")
626
+ if 'confidence' in result:
627
+ print(f"📊 Confidence: {result['confidence']:.1%}")
628
+ if 'probabilities' in result:
629
+ print(f"📊 Probabilities:")
630
+ for cls, prob in sorted(result['probabilities'].items(), key=lambda x: x[1], reverse=True):
631
+ bar = "█" * int(prob * 30)
632
+ print(f" {cls:>15}: {prob:.1%} {bar}")
633
+
634
+ return result
635
+
636
+
637
+ def main():
638
+ parser = argparse.ArgumentParser(description="Make predictions with your trained model")
639
+ parser.add_argument('--model', type=str, default='model.pkl', help='Path to model file')
640
+ parser.add_argument('--dl-model', type=str, default='deep_learning_model.pkl', help='Path to DL model file')
641
+ parser.add_argument('--mode', type=str, default='auto', choices=['auto', 'traditional', 'dl'], help='Prediction mode')
642
+ parser.add_argument('--json', type=str, default=None, help='JSON string with input data')
643
+ parser.add_argument('--csv', type=str, default=None, help='CSV file for batch predictions')
644
+ parser.add_argument('--output', type=str, default=None, help='Output CSV for batch predictions')
645
+ parser.add_argument('--interactive', action='store_true', help='Interactive mode')
646
+ args = parser.parse_args()
647
+
648
+ # Auto-detect which model to use based on BEST_MODE from training
649
+ use_dl = False
650
+ dl_state = None
651
+ state = None
652
+
653
+ # Use DL if: explicitly requested, or BEST_MODE is deep_learning, or DL model exists
654
+ if args.mode == 'dl' or (args.mode == 'auto' and BEST_MODE == 'deep_learning') or (args.mode == 'auto' and os.path.exists(args.dl_model) and not os.path.exists(args.model)):
655
+ dl_state = load_dl_model(args.dl_model)
656
+ if dl_state:
657
+ use_dl = True
658
+
659
+ if not use_dl:
660
+ state = load_model(args.model)
661
+ if state is None:
662
+ # Last resort: try DL model
663
+ dl_state = load_dl_model(args.dl_model)
664
+ if dl_state:
665
+ use_dl = True
666
+ else:
667
+ return
668
+
669
+ if args.json:
670
+ input_data = json.loads(args.json)
671
+ if use_dl:
672
+ result = predict_single_dl(dl_state, input_data)
673
+ else:
674
+ result = predict_single(state, input_data)
675
+ print(f"\n🎯 Prediction: {result['prediction']}")
676
+ if 'confidence' in result:
677
+ print(f"📊 Confidence: {result['confidence']:.1%}")
678
+ print(f"\n📋 Full result: {json.dumps(result, indent=2, default=str)}")
679
+ elif args.csv:
680
+ if use_dl:
681
+ # Batch DL predictions
682
+ df = pd.read_csv(args.csv)
683
+ print(f"\n📊 Predicting for {len(df)} rows (DL mode)...")
684
+ predictions = []
685
+ for _, row in df.iterrows():
686
+ result = predict_single_dl(dl_state, row.to_dict())
687
+ predictions.append(result['prediction'])
688
+ df['prediction'] = predictions
689
+ output_path = args.output or args.csv.replace('.csv', '_predictions.csv')
690
+ df.to_csv(output_path, index=False)
691
+ print(f"💾 Predictions saved to: {output_path}")
692
+ else:
693
+ predict_batch(state, args.csv, args.output)
694
+ else:
695
+ interactive_mode(state, use_dl=use_dl)
696
+
697
+
698
+ # =========================================================================
699
+ # Example (using YOUR actual feature names):
700
+ #
701
+ # from predict import load_model, predict_single
702
+ # state = load_model('model.pkl')
703
+ # result = predict_single(state, {
704
+ # 'battery_power': 1238.51, # numeric, range [510.0 .. 1987.0]
705
+ # 'blue': 0.49, # numeric, range [0.0 .. 1.0]
706
+ # 'clock_speed': 1.52, # numeric, range [0.5 .. 3.0]
707
+ # 'dual_sim': 0.51, # numeric, range [0.0 .. 1.0]
708
+ # 'fc': 4.29, # numeric, range [0.0 .. 16.0]
709
+ # 'four_g': 0.52, # numeric, range [0.0 .. 1.0]
710
+ # 'int_memory': 32.05, # numeric, range [2.0 .. 64.0]
711
+ # 'm_dep': 0.50, # numeric, range [0.1 .. 1.0]
712
+ # 'mobile_wt': 140.24, # numeric, range [80.0 .. 199.0]
713
+ # 'n_cores': 4.52, # numeric, range [1.0 .. 8.0]
714
+ # 'pc': 9.92, # numeric, range [0.0 .. 20.0]
715
+ # 'px_height': 644.46, # numeric, range [15.0 .. 1791.01]
716
+ # 'ram': 2124.26, # numeric, range [296.99 .. 3958.01]
717
+ # 'sc_h': 12.31, # numeric, range [5.0 .. 19.0]
718
+ # 'sc_w': 5.76, # numeric, range [0.0 .. 17.0]
719
+ # 'three_g': 1.00, # numeric, range [1.0 .. 1.0]
720
+ # 'touch_screen': 0.50, # numeric, range [0.0 .. 1.0]
721
+ # 'wifi': 0.51, # numeric, range [0.0 .. 1.0]
722
+ # })
723
+ # print(result['prediction'])
724
+ # =========================================================================
725
+
726
+
727
+ if __name__ == '__main__':
728
+ main()
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/requirements.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ flask>=2.2.0
2
+ imbalanced-learn>=0.10.0
3
+ joblib>=1.1.0
4
+ matplotlib>=3.5.0
5
+ numpy>=1.21.0
6
+ pandas>=1.3.0
7
+ scikit-learn>=1.0.0
8
+ scipy>=1.7.0
9
+ seaborn>=0.12.0
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/train_traditional.py ADDED
@@ -0,0 +1,894 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🚀 Traditional ML Training Script — YOUR DataVision AutoML Results
3
+ =====================================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ ╔═══════════════════════════════════════════════════════════════════════╗
7
+ ║ 📦 model.pkl contains your ALREADY TRAINED model with the metrics ║
8
+ ║ you saw in the AutoML UI. ║
9
+ ║ ║
10
+ ║ ▶ python evaluate.py — See your EXACT AutoML results ║
11
+ ║ ▶ python predict.py — Make predictions with your model ║
12
+ ║ ║
13
+ ║ This script is for RETRAINING only. Results may differ slightly ║
14
+ ║ due to train/test random split, even with same hyperparameters. ║
15
+ ╚═══════════════════════════════════════════════════════════════════════╝
16
+
17
+ YOUR AUTOML TRAINING RESULTS:
18
+ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
19
+ 🏆 WINNING MODEL : RandomForest
20
+ 🎯 Target : price_range
21
+ 📊 Task : classification
22
+ 📁 Features : 20 columns
23
+
24
+ YOUR METRICS:
25
+ # accuracy: 0.8925
26
+ # f1: 0.8929
27
+ # precision: 0.8937
28
+ # recall: 0.8925
29
+ # roc_auc: 0.9796
30
+
31
+ YOUR LEADERBOARD (what AutoML trained):
32
+ # 1. ?: 0.8925
33
+
34
+ FEATURE INFO:
35
+ # battery_power: numeric (min=510.0, max=1987.0, mean=1238.513)
36
+ # blue: numeric (min=0.0, max=1.0, mean=0.495)
37
+ # clock_speed: numeric (min=0.5, max=3.0, mean=1.52225)
38
+ # dual_sim: numeric (min=0.0, max=1.0, mean=0.5095)
39
+ # fc: numeric (min=0.0, max=16.0, mean=4.294)
40
+ # four_g: numeric (min=0.0, max=1.0, mean=0.5215)
41
+ # int_memory: numeric (min=2.0, max=64.0, mean=32.0465)
42
+ # m_dep: numeric (min=0.1, max=1.0, mean=0.50175)
43
+ # mobile_wt: numeric (min=80.0, max=199.0, mean=140.2395)
44
+ # n_cores: numeric (min=1.0, max=8.0, mean=4.5205)
45
+
46
+ Usage:
47
+ # VERIFY YOUR AUTOML RESULTS:
48
+ python evaluate.py # Shows exact AutoML metrics
49
+
50
+ # MAKE PREDICTIONS:
51
+ python predict.py --json '{"col1": value}' # Your trained model
52
+
53
+ # RETRAIN (if needed):
54
+ python train_traditional.py # Train ONLY RandomForest
55
+ python train_traditional.py --all # Train ALL algorithms
56
+ """
57
+
58
+ import os
59
+ import pickle
60
+ import argparse
61
+ import time
62
+ import warnings
63
+ import numpy as np
64
+ import pandas as pd
65
+ from scipy import stats as scipy_stats
66
+ from sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score
67
+ from sklearn.preprocessing import LabelEncoder, RobustScaler, StandardScaler
68
+ from sklearn.feature_selection import VarianceThreshold
69
+ from sklearn.impute import KNNImputer
70
+ from sklearn.metrics import (
71
+ accuracy_score, precision_score, recall_score, f1_score,
72
+ r2_score, mean_absolute_error, mean_squared_error,
73
+ classification_report, confusion_matrix
74
+ )
75
+ from sklearn.ensemble import (
76
+ RandomForestClassifier, RandomForestRegressor,
77
+ GradientBoostingClassifier, GradientBoostingRegressor,
78
+ ExtraTreesClassifier, ExtraTreesRegressor,
79
+ HistGradientBoostingClassifier, HistGradientBoostingRegressor,
80
+ AdaBoostClassifier, AdaBoostRegressor,
81
+ BaggingClassifier, BaggingRegressor,
82
+ )
83
+ from sklearn.linear_model import (
84
+ LogisticRegression, Ridge, Lasso, ElasticNet,
85
+ SGDClassifier, SGDRegressor, RidgeClassifier,
86
+ PassiveAggressiveClassifier, PassiveAggressiveRegressor,
87
+ BayesianRidge
88
+ )
89
+ from sklearn.svm import SVC, SVR
90
+ from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
91
+ from sklearn.naive_bayes import GaussianNB, BernoulliNB
92
+ from sklearn.neural_network import MLPClassifier, MLPRegressor
93
+ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
94
+
95
+ warnings.filterwarnings('ignore')
96
+
97
+ HAS_XGB = False
98
+ HAS_LGB = False
99
+ HAS_CB = False
100
+ HAS_SMOTE = False
101
+
102
+ try:
103
+ import xgboost as xgb
104
+ HAS_XGB = True
105
+ except ImportError:
106
+ pass
107
+ try:
108
+ import lightgbm as lgb
109
+ HAS_LGB = True
110
+ except ImportError:
111
+ pass
112
+ try:
113
+ from catboost import CatBoostClassifier, CatBoostRegressor
114
+ HAS_CB = True
115
+ except ImportError:
116
+ pass
117
+ try:
118
+ from imblearn.over_sampling import SMOTE
119
+ HAS_SMOTE = True
120
+ except ImportError:
121
+ pass
122
+
123
+ TARGET_COLUMN = 'price_range'
124
+ TASK_TYPE = 'classification'
125
+
126
+ # YOUR WINNING MODEL FROM AUTOML
127
+ BEST_MODEL_NAME = 'RandomForest'
128
+
129
+
130
+ # ============================================================================
131
+ # STEP 1: DATA CLEANING (mirrors ProductionDataCleaner.clean())
132
+ # ============================================================================
133
+
134
+ def clean_data(df):
135
+ """
136
+ Mirrors DataVision's ProductionDataCleaner pipeline:
137
+ 1. Remove useless columns (IDs, URLs, high-cardinality strings)
138
+ 2. Remove exact duplicates
139
+ 3. Strip whitespace, replace 'nan'/'None'/'' with NaN
140
+ 4. Smart type conversion (parse $, M/K units, commas)
141
+ 5. KNN imputation for numerics (median fallback)
142
+ 6. Mode fill for categoricals
143
+ 7. Replace inf/-inf with median
144
+ 8. Remove constant columns (nunique <= 1)
145
+ 9. IQR-based outlier capping (Q1-3*IQR to Q3+3*IQR, clipped to [1st, 99th] percentile)
146
+ 10. Final NaN fill to 0
147
+ """
148
+ print("\n🧹 Step 1: Data Cleaning (ProductionDataCleaner)")
149
+ print("-" * 50)
150
+ original_shape = df.shape
151
+
152
+ # Remove useless columns
153
+ id_patterns = ['unnamed', 'index', '_id', 'guid', 'uuid', 'url', 'link', 'href', 'path']
154
+ cols_to_drop = []
155
+ for col in df.columns:
156
+ col_lower = col.lower().strip()
157
+ if any(pat in col_lower for pat in id_patterns):
158
+ if col != TARGET_COLUMN:
159
+ cols_to_drop.append(col)
160
+ elif df[col].dtype == 'object' and col != TARGET_COLUMN:
161
+ unique_ratio = df[col].nunique() / max(len(df), 1)
162
+ if unique_ratio > 0.9:
163
+ cols_to_drop.append(col)
164
+ if cols_to_drop:
165
+ df = df.drop(columns=cols_to_drop, errors='ignore')
166
+ print(f" Dropped {len(cols_to_drop)} useless columns: {cols_to_drop[:5]}")
167
+
168
+ # Remove exact duplicates
169
+ before = len(df)
170
+ df = df.drop_duplicates()
171
+ if len(df) < before:
172
+ print(f" Removed {before - len(df)} duplicate rows")
173
+
174
+ # Strip whitespace + replace string NaN variants
175
+ for col in df.select_dtypes(include=['object']).columns:
176
+ df[col] = df[col].str.strip()
177
+ df[col] = df[col].replace(
178
+ ['nan', 'NaN', 'None', 'null', 'NULL', 'none', 'NA', 'N/A', ''],
179
+ np.nan
180
+ )
181
+
182
+ # Smart type conversion (parse $, M/K, commas)
183
+ for col in df.select_dtypes(include=['object']).columns:
184
+ if col == TARGET_COLUMN:
185
+ continue
186
+ sample = df[col].dropna().head(20)
187
+ if len(sample) == 0:
188
+ continue
189
+ converted = 0
190
+ for val in sample:
191
+ try:
192
+ v = str(val).replace('$', '').replace(',', '').strip()
193
+ if v.upper().endswith('M'):
194
+ float(v[:-1])
195
+ elif v.upper().endswith('K'):
196
+ float(v[:-1])
197
+ elif v.endswith('+'):
198
+ float(v[:-1])
199
+ else:
200
+ float(v)
201
+ converted += 1
202
+ except:
203
+ pass
204
+ if converted / max(len(sample), 1) > 0.6:
205
+ def parse_value(val):
206
+ if pd.isna(val):
207
+ return np.nan
208
+ v = str(val).replace('$', '').replace(',', '').strip()
209
+ try:
210
+ if v.upper().endswith('M'):
211
+ return float(v[:-1]) * 1_000_000
212
+ elif v.upper().endswith('K'):
213
+ return float(v[:-1]) * 1_000
214
+ elif v.endswith('+'):
215
+ return float(v[:-1])
216
+ return float(v)
217
+ except:
218
+ return np.nan
219
+ df[col] = df[col].apply(parse_value)
220
+ print(f" Converted {col} to numeric (parsed special formats)")
221
+
222
+ # KNN imputation for numeric columns
223
+ numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
224
+ if TARGET_COLUMN in numeric_cols:
225
+ numeric_cols.remove(TARGET_COLUMN)
226
+ if numeric_cols and df[numeric_cols].isna().any().any():
227
+ try:
228
+ imputer = KNNImputer(n_neighbors=5, weights='distance')
229
+ df[numeric_cols] = imputer.fit_transform(df[numeric_cols])
230
+ print(f" KNN imputed {len(numeric_cols)} numeric columns")
231
+ except:
232
+ for col in numeric_cols:
233
+ df[col] = df[col].fillna(df[col].median())
234
+ print(f" Median-filled {len(numeric_cols)} numeric columns (KNN fallback)")
235
+
236
+ # Mode fill for categoricals
237
+ for col in df.select_dtypes(include=['object', 'category']).columns:
238
+ if col == TARGET_COLUMN:
239
+ continue
240
+ mode = df[col].mode()
241
+ df[col] = df[col].fillna(mode.iloc[0] if len(mode) > 0 else '_MISSING_')
242
+
243
+ # Replace inf/-inf with median
244
+ for col in numeric_cols:
245
+ if np.isinf(df[col]).any():
246
+ med = df[col].replace([np.inf, -np.inf], np.nan).median()
247
+ df[col] = df[col].replace([np.inf, -np.inf], med)
248
+
249
+ # Remove constant columns
250
+ const_cols = [col for col in df.columns if col != TARGET_COLUMN and df[col].nunique() <= 1]
251
+ if const_cols:
252
+ df = df.drop(columns=const_cols)
253
+ print(f" Removed {len(const_cols)} constant columns")
254
+
255
+ # IQR-based outlier capping (same as DataVision)
256
+ capped_count = 0
257
+ for col in df.select_dtypes(include=[np.number]).columns:
258
+ if col == TARGET_COLUMN:
259
+ continue
260
+ Q1 = df[col].quantile(0.25)
261
+ Q3 = df[col].quantile(0.75)
262
+ IQR = Q3 - Q1
263
+ p1 = df[col].quantile(0.01)
264
+ p99 = df[col].quantile(0.99)
265
+ lower = max(p1, Q1 - 3 * IQR)
266
+ upper = min(p99, Q3 + 3 * IQR)
267
+ before_clip = ((df[col] < lower) | (df[col] > upper)).sum()
268
+ if before_clip > 0:
269
+ df[col] = df[col].clip(lower=lower, upper=upper)
270
+ capped_count += before_clip
271
+ if capped_count:
272
+ print(f" Capped {capped_count} outlier values (IQR method)")
273
+
274
+ # Final NaN fill
275
+ for col in df.select_dtypes(include=[np.number]).columns:
276
+ df[col] = df[col].fillna(0)
277
+
278
+ print(f" Clean: {original_shape} → {df.shape}")
279
+ return df
280
+
281
+
282
+ # ============================================================================
283
+ # STEP 2: FEATURE ENGINEERING (mirrors ProductionFeatureEngineer)
284
+ # ============================================================================
285
+
286
+ def engineer_features(X, mode='fast'):
287
+ """
288
+ Mirrors DataVision's ProductionFeatureEngineer.fit_transform():
289
+ 1. Detect column types (numeric, categorical, text, datetime)
290
+ 2. Log-transform skewed numerics (skewness > 2)
291
+ 3. RobustScaler for numeric features
292
+ 4. Polynomial features (squared + interactions for top columns)
293
+ 5. Categorical: binary→LabelEncoder, low-card→one-hot, high-card→label+frequency
294
+ 6. VarianceThreshold (remove near-constant features, threshold=0.01)
295
+ """
296
+ print("\n🔧 Step 2: Feature Engineering (ProductionFeatureEngineer)")
297
+ print("-" * 50)
298
+
299
+ numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
300
+ categorical_cols = X.select_dtypes(include=['object', 'category']).columns.tolist()
301
+
302
+ label_encoders = {}
303
+
304
+ # --- Numeric feature engineering ---
305
+ # Log transform skewed features (skewness > 2, non-negative only)
306
+ log_transformed = []
307
+ for col in numeric_cols:
308
+ if (X[col] >= 0).all():
309
+ skew = X[col].skew()
310
+ if abs(skew) > 2:
311
+ X[f'{col}_log'] = np.log1p(X[col])
312
+ log_transformed.append(col)
313
+ if log_transformed:
314
+ print(f" Log-transformed {len(log_transformed)} skewed features")
315
+
316
+ # Scale with RobustScaler (same as DataVision)
317
+ numeric_cols_updated = X.select_dtypes(include=[np.number]).columns.tolist()
318
+ scaler = RobustScaler()
319
+ if numeric_cols_updated:
320
+ X[numeric_cols_updated] = scaler.fit_transform(X[numeric_cols_updated])
321
+
322
+ # Polynomial features (top 4 numeric columns — squared + interactions)
323
+ if 2 <= len(numeric_cols) <= 10:
324
+ top_n = min(4, len(numeric_cols))
325
+ top_cols = numeric_cols[:top_n]
326
+ for col in top_cols:
327
+ X[f'{col}_sq'] = X[col] ** 2
328
+ for i in range(len(top_cols)):
329
+ for j in range(i + 1, len(top_cols)):
330
+ X[f'{top_cols[i]}_x_{top_cols[j]}'] = X[top_cols[i]] * X[top_cols[j]]
331
+ print(f" Added polynomial features for top {top_n} numeric columns")
332
+
333
+ # --- Categorical feature engineering ---
334
+ for col in categorical_cols:
335
+ nuniq = X[col].nunique()
336
+ if nuniq <= 2:
337
+ # Binary: LabelEncoder
338
+ le = LabelEncoder()
339
+ X[col] = le.fit_transform(X[col].astype(str))
340
+ label_encoders[col] = le
341
+ elif nuniq <= 10:
342
+ # Low cardinality: one-hot encoding (no drop_first — same as DataVision)
343
+ dummies = pd.get_dummies(X[col], prefix=col, drop_first=False)
344
+ X = pd.concat([X.drop(columns=[col]), dummies], axis=1)
345
+ else:
346
+ # High cardinality: LabelEncoder + frequency encoding
347
+ le = LabelEncoder()
348
+ X[f'{col}_label'] = le.fit_transform(X[col].astype(str))
349
+ freq = X[col].value_counts(normalize=True)
350
+ X[f'{col}_freq'] = X[col].map(freq)
351
+ label_encoders[col] = le
352
+ X = X.drop(columns=[col])
353
+
354
+ # Force all to float
355
+ for col in X.columns:
356
+ try:
357
+ X[col] = X[col].astype(float)
358
+ except:
359
+ le = LabelEncoder()
360
+ X[col] = le.fit_transform(X[col].astype(str)).astype(float)
361
+
362
+ # Handle NaN/Inf
363
+ X = X.replace([np.inf, -np.inf], 0)
364
+ X = X.fillna(0)
365
+
366
+ # VarianceThreshold (remove near-constant features)
367
+ try:
368
+ vt = VarianceThreshold(threshold=0.01)
369
+ cols_before = X.columns.tolist()
370
+ X_filtered = pd.DataFrame(vt.fit_transform(X), columns=X.columns[vt.get_support()], index=X.index)
371
+ removed = len(cols_before) - X_filtered.shape[1]
372
+ if removed > 0:
373
+ print(f" VarianceThreshold removed {removed} near-constant features")
374
+ X = X_filtered
375
+ except:
376
+ pass
377
+
378
+ print(f" Final features: {X.shape[1]}")
379
+ return X, label_encoders, scaler
380
+
381
+
382
+ # ============================================================================
383
+ # STEP 3: YOUR EXACT TRAINED MODEL
384
+ # ============================================================================
385
+
386
+ # These are the EXACT hyperparameters from YOUR trained model
387
+ ACTUAL_MODEL_CLASS = 'RandomForestClassifier'
388
+ ACTUAL_MODEL_PARAMS = {'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': 'balanced', 'criterion': 'gini', 'max_depth': 10, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'monotonic_cst': None, 'n_estimators': 150, 'random_state': 42}
389
+
390
+ def get_exact_trained_model():
391
+ """Returns YOUR exact model with the SAME hyperparameters as AutoML used.
392
+
393
+ This ensures retraining produces the SAME results as your AutoML session.
394
+ """
395
+ params = ACTUAL_MODEL_PARAMS.copy()
396
+
397
+ # Remove non-constructor / problematic params that cause crashes
398
+ for bad_key in ['oob_score', 'warm_start', 'verbose', 'callbacks',
399
+ 'early_stopping_rounds', 'use_label_encoder', 'gpu_id',
400
+ 'predictor', 'enable_categorical', 'feature_types',
401
+ 'validate_parameters', 'nthread', 'device',
402
+ 'n_jobs', 'silent', 'importance_type',
403
+ 'cat_features', 'text_features', 'task_type',
404
+ 'multi_strategy', 'max_cat_to_onehot', 'max_cat_threshold',
405
+ 'num_parallel_tree', 'monotone_constraints',
406
+ 'interaction_constraints', 'grow_policy', 'max_leaves',
407
+ 'process_type', 'updater', 'refresh_leaf', 'max_bin',
408
+ 'bootstrap_type', 'devices', 'embedding_features']:
409
+ params.pop(bad_key, None)
410
+ # Remove private params
411
+ params = {k: v for k, v in params.items() if not k.startswith('_')}
412
+
413
+
414
+ return {'RandomForest': RandomForestClassifier(**params)}
415
+
416
+
417
+ def get_single_model(model_name, n_samples):
418
+ """Get ONLY your winning model.
419
+
420
+ Uses get_exact_trained_model() to get YOUR exact hyperparameters.
421
+ """
422
+ # First try to use exact trained model params
423
+ if model_name == BEST_MODEL_NAME or model_name == 'RandomForest':
424
+ return get_exact_trained_model()
425
+
426
+ # Otherwise fall back to generic models
427
+ is_large = n_samples > 50000
428
+ all_models = get_models(n_samples, mode='fast')
429
+
430
+ # Try exact match first
431
+ if model_name in all_models:
432
+ return {model_name: all_models[model_name]}
433
+
434
+ # Try case-insensitive match
435
+ model_name_lower = model_name.lower().replace('_', '').replace(' ', '')
436
+ for k, v in all_models.items():
437
+ if k.lower().replace('_', '').replace(' ', '') == model_name_lower:
438
+ return {k: v}
439
+
440
+ # Check in ultra models too
441
+ all_models = get_models(n_samples, mode='ultra')
442
+ if model_name in all_models:
443
+ return {model_name: all_models[model_name]}
444
+ for k, v in all_models.items():
445
+ if k.lower().replace('_', '').replace(' ', '') == model_name_lower:
446
+ return {k: v}
447
+
448
+ # Fallback to exact trained model
449
+ print(f" ⚠️ Model '{model_name}' not found, using your trained model")
450
+ return get_exact_trained_model()
451
+
452
+
453
+ def get_models(n_samples, mode='fast'):
454
+ """Returns the EXACT algorithms & hyperparameters used by DataVision AutoML.
455
+
456
+ Mirrors ProductionModelTrainer.get_models() with the same conditional
457
+ logic for dataset size and mode.
458
+ """
459
+ is_large = n_samples > 50000
460
+ is_ultra = mode == 'ultra'
461
+
462
+
463
+ models = {
464
+ 'LogisticRegression': LogisticRegression(
465
+ max_iter=2000, C=0.1, random_state=42, n_jobs=1,
466
+ class_weight='balanced'
467
+ ),
468
+ 'RandomForest': RandomForestClassifier(
469
+ n_estimators=100 if is_large else (150 if not is_ultra else 250),
470
+ max_depth=8 if is_large else (12 if not is_ultra else 18),
471
+ min_samples_split=4, random_state=42, n_jobs=-1,
472
+ class_weight='balanced'
473
+ ),
474
+ 'GradientBoosting': GradientBoostingClassifier(
475
+ n_estimators=80 if is_large else 120,
476
+ max_depth=5 if is_large else 6,
477
+ learning_rate=0.1, random_state=42, subsample=0.8
478
+ ),
479
+ 'HistGradientBoosting': HistGradientBoostingClassifier(
480
+ max_iter=100 if is_large else (150 if not is_ultra else 250),
481
+ max_depth=6 if is_large else (8 if not is_ultra else 12),
482
+ learning_rate=0.1, random_state=42, class_weight='balanced'
483
+ ),
484
+ 'GaussianNB': GaussianNB(),
485
+ 'ExtraTrees': ExtraTreesClassifier(
486
+ n_estimators=80 if is_large else (100 if not is_ultra else 200),
487
+ max_depth=8 if is_large else (10 if not is_ultra else 15),
488
+ random_state=42, n_jobs=-1, class_weight='balanced'
489
+ ),
490
+ }
491
+
492
+ if not is_large:
493
+ models['KNN'] = KNeighborsClassifier(n_neighbors=7, weights='distance', n_jobs=-1)
494
+
495
+ if HAS_XGB:
496
+ models['XGBoost'] = xgb.XGBClassifier(
497
+ n_estimators=100 if is_large else (150 if not is_ultra else 300),
498
+ max_depth=6 if is_large else (7 if not is_ultra else 10),
499
+ learning_rate=0.1 if not is_ultra else 0.05,
500
+ subsample=0.8, colsample_bytree=0.8,
501
+ random_state=42, n_jobs=-1, verbosity=0,
502
+ eval_metric='mlogloss',
503
+ tree_method='hist' if is_large else 'auto',
504
+ scale_pos_weight=10
505
+ )
506
+ if HAS_LGB:
507
+ models['LightGBM'] = lgb.LGBMClassifier(
508
+ n_estimators=100 if is_large else (150 if not is_ultra else 300),
509
+ max_depth=6 if is_large else (7 if not is_ultra else 10),
510
+ learning_rate=0.1 if not is_ultra else 0.05,
511
+ num_leaves=31 if not is_ultra else 50,
512
+ subsample=0.8, colsample_bytree=0.8,
513
+ random_state=42, n_jobs=-1, verbose=-1,
514
+ class_weight='balanced'
515
+ )
516
+ if HAS_CB:
517
+ if is_ultra:
518
+ models['CatBoost'] = CatBoostClassifier(
519
+ iterations=150 if is_large else 300, depth=6 if is_large else 10,
520
+ learning_rate=0.05 if is_large else 0.03,
521
+ l2_leaf_reg=3, random_strength=1, bagging_temperature=0.5,
522
+ random_state=42, verbose=False,
523
+ auto_class_weights='Balanced', allow_writing_files=False
524
+ )
525
+ else:
526
+ models['CatBoost'] = CatBoostClassifier(
527
+ iterations=100, depth=6, learning_rate=0.1,
528
+ random_state=42, verbose=False,
529
+ auto_class_weights='Balanced', allow_writing_files=False
530
+ )
531
+
532
+ # Ultra mode adds more algorithms
533
+ if is_ultra:
534
+ if not is_large:
535
+ models['SVM_RBF'] = SVC(kernel='rbf', C=1.0, gamma='scale', probability=True, random_state=42, class_weight='balanced')
536
+ models['SVM_Linear'] = SVC(kernel='linear', C=0.5, probability=True, random_state=42, class_weight='balanced')
537
+ models['RidgeClassifier'] = RidgeClassifier(alpha=1.0, random_state=42, class_weight='balanced')
538
+ models['MLP'] = MLPClassifier(
539
+ hidden_layer_sizes=(128, 64) if is_large else (256, 128, 64, 32),
540
+ max_iter=500 if is_large else 1500,
541
+ learning_rate='adaptive', early_stopping=True,
542
+ batch_size=256 if is_large else 'auto',
543
+ validation_fraction=0.1, random_state=42
544
+ )
545
+ models['AdaBoost'] = AdaBoostClassifier(n_estimators=100 if is_large else 150, learning_rate=0.5, random_state=42)
546
+ models['Bagging'] = BaggingClassifier(
547
+ n_estimators=50 if is_large else 100,
548
+ max_samples=0.5 if is_large else 0.8,
549
+ max_features=0.8, random_state=42, n_jobs=-1
550
+ )
551
+ models['SGD'] = SGDClassifier(loss='log_loss', max_iter=1500, random_state=42, class_weight='balanced', early_stopping=True)
552
+ models['PassiveAggressive'] = PassiveAggressiveClassifier(max_iter=1500, random_state=42, class_weight='balanced')
553
+ models['DecisionTree'] = DecisionTreeClassifier(max_depth=12 if is_large else 18, min_samples_split=3, random_state=42, class_weight='balanced')
554
+ models['BernoulliNB'] = BernoulliNB(alpha=0.5)
555
+
556
+ return models
557
+
558
+
559
+ # ============================================================================
560
+ # STEP 4: TRAINING (mirrors ProductionModelTrainer.train_all())
561
+ # ============================================================================
562
+
563
+ def apply_smote(X_train, y_train):
564
+ """Apply SMOTE for imbalanced classification (same as DataVision).
565
+
566
+ Only applied when:
567
+ - imbalance_ratio > 1.5
568
+ - min class count >= 6
569
+ Falls back to class_weight='balanced' if SMOTE fails.
570
+ """
571
+ if not HAS_SMOTE or TASK_TYPE != 'classification':
572
+ return X_train, y_train, False
573
+
574
+ class_counts = pd.Series(y_train).value_counts()
575
+ max_count = class_counts.max()
576
+ min_count = class_counts.min()
577
+ imbalance_ratio = max_count / max(min_count, 1)
578
+
579
+ if imbalance_ratio <= 1.5 or min_count < 6:
580
+ return X_train, y_train, False
581
+
582
+ try:
583
+ k_neighbors = min(5, min_count - 1)
584
+ n_classes = len(class_counts)
585
+
586
+ if n_classes == 2:
587
+ if imbalance_ratio > 100:
588
+ sampling_strategy = 0.1
589
+ elif imbalance_ratio > 20:
590
+ sampling_strategy = 0.25
591
+ else:
592
+ sampling_strategy = 1.0
593
+ else:
594
+ sampling_strategy = 'not majority' if imbalance_ratio > 20 else 'auto'
595
+
596
+ smote = SMOTE(k_neighbors=k_neighbors, sampling_strategy=sampling_strategy, random_state=42)
597
+ X_resampled, y_resampled = smote.fit_resample(X_train, y_train)
598
+ print(f" ⚖️ SMOTE applied: {len(X_train)} → {len(X_resampled)} samples (ratio: {imbalance_ratio:.1f}x)")
599
+ return X_resampled, y_resampled, True
600
+ except Exception as e:
601
+ print(f" ⚠️ SMOTE failed ({e}), using class_weight='balanced'")
602
+ return X_train, y_train, False
603
+
604
+
605
+ def train_and_evaluate(X_train, X_test, y_train, y_test, mode='fast', selected_models=None):
606
+ """Train models with real DataVision scoring logic.
607
+
608
+ Args:
609
+ selected_models: Optional dict of {name: model} to train only specific models.
610
+ If None, trains all models for the specified mode.
611
+
612
+ Scoring:
613
+ - Classification (balanced): F1-macro
614
+ - Classification (imbalanced >5x): minority_recall*0.6 + f1_macro*0.4
615
+ - Regression: R² score
616
+
617
+ Cross-validation:
618
+ - Fast: 3-fold StratifiedKFold/KFold
619
+ - Ultra: 5-fold
620
+ """
621
+ is_clf = TASK_TYPE == 'classification'
622
+ is_ultra = mode == 'ultra'
623
+
624
+ # Apply SMOTE (classification only)
625
+ X_train_sm, y_train_sm, smote_applied = apply_smote(X_train, y_train)
626
+
627
+ # Use selected_models if provided, otherwise get all models for the mode
628
+ if selected_models is not None:
629
+ models = selected_models
630
+ else:
631
+ models = get_models(len(X_train), mode)
632
+ results = []
633
+ best_model = None
634
+ best_score = -999
635
+ best_name = ""
636
+
637
+ # Check imbalance for scoring
638
+ if is_clf:
639
+ class_counts = pd.Series(y_train).value_counts()
640
+ imbalance_ratio = class_counts.max() / max(class_counts.min(), 1)
641
+ is_imbalanced = imbalance_ratio > 5
642
+ minority_class = class_counts.idxmin()
643
+
644
+ # Cross-validation setup (same as DataVision)
645
+ cv_folds = 3 if mode == 'fast' else 5
646
+ if is_clf:
647
+ cv = StratifiedKFold(n_splits=cv_folds, shuffle=True, random_state=42)
648
+ else:
649
+ cv = KFold(n_splits=cv_folds, shuffle=True, random_state=42)
650
+
651
+ print(f"\n🏋️ Training {len(models)} model(s) ({'ALL' if selected_models is None else 'SELECTED'} mode, {cv_folds}-fold CV)...")
652
+ print("=" * 70)
653
+
654
+ for name, model in models.items():
655
+ try:
656
+ t0 = time.time()
657
+ model.fit(X_train_sm, y_train_sm)
658
+ y_pred = model.predict(X_test)
659
+ elapsed = time.time() - t0
660
+
661
+ if is_clf:
662
+ f1_macro = f1_score(y_test, y_pred, average='macro', zero_division=0)
663
+
664
+ if is_imbalanced and len(pd.Series(y_train).unique()) == 2:
665
+ # DataVision's real scoring for imbalanced binary:
666
+ # minority_recall * 0.6 + f1_macro * 0.4
667
+ from sklearn.metrics import recall_score as rs
668
+ minority_recall = rs(y_test, y_pred, pos_label=minority_class, zero_division=0)
669
+ score = minority_recall * 0.6 + f1_macro * 0.4
670
+ else:
671
+ score = f1_macro
672
+
673
+ metrics = {
674
+ 'accuracy': accuracy_score(y_test, y_pred),
675
+ 'precision': precision_score(y_test, y_pred, average='weighted', zero_division=0),
676
+ 'recall': recall_score(y_test, y_pred, average='weighted', zero_division=0),
677
+ 'f1_weighted': f1_score(y_test, y_pred, average='weighted', zero_division=0),
678
+ 'f1_macro': f1_macro,
679
+ 'score': score,
680
+ }
681
+ else:
682
+ score = r2_score(y_test, y_pred)
683
+ metrics = {
684
+ 'r2': score,
685
+ 'mae': mean_absolute_error(y_test, y_pred),
686
+ 'rmse': np.sqrt(mean_squared_error(y_test, y_pred)),
687
+ 'score': score,
688
+ }
689
+
690
+ # Cross-validation
691
+ try:
692
+ cv_scoring = 'f1_weighted' if is_clf else 'r2'
693
+ cv_scores = cross_val_score(model, X_train_sm, y_train_sm, cv=cv, scoring=cv_scoring)
694
+ metrics['cv_mean'] = cv_scores.mean()
695
+ metrics['cv_std'] = cv_scores.std()
696
+ except:
697
+ metrics['cv_mean'] = score
698
+ metrics['cv_std'] = 0
699
+
700
+ results.append({
701
+ 'name': name, 'score': score, 'metrics': metrics, 'model': model,
702
+ 'y_pred': y_pred, 'time': elapsed, 'cv_mean': metrics.get('cv_mean', score)
703
+ })
704
+
705
+ marker = "🏆" if score > best_score else " "
706
+ cv_str = f"CV: {metrics.get('cv_mean', 0):.4f}±{metrics.get('cv_std', 0):.3f}"
707
+ print(f" {marker} {name:.<30} Score: {score:.4f} | {cv_str} ({elapsed:.1f}s)")
708
+
709
+ if score > best_score:
710
+ best_score = score
711
+ best_model = model
712
+ best_name = name
713
+
714
+ except Exception as e:
715
+ print(f" ❌ {name:.<30} Failed: {str(e)[:60]}")
716
+
717
+ # Sort results by score
718
+ results.sort(key=lambda x: x['score'], reverse=True)
719
+
720
+ print("=" * 70)
721
+
722
+ if not results:
723
+ print("\n❌ All models failed to train!")
724
+ print(" Try: python train_traditional.py --all (to try all algorithms)")
725
+ return None, '', {}, []
726
+
727
+ print(f"\n🏆 Best Model: {best_name} (Score: {best_score:.4f})")
728
+
729
+ best_result = results[0]
730
+ if is_clf:
731
+ print(f"\n📊 Classification Report:")
732
+ print(classification_report(y_test, best_result['y_pred']))
733
+ else:
734
+ print(f"\n📊 Regression Metrics:")
735
+ for k, v in best_result['metrics'].items():
736
+ if k not in ('score', 'cv_mean', 'cv_std'):
737
+ print(f" {k.upper()}: {v:.4f}")
738
+
739
+ # Overfitting check (same as DataVision's Production Intelligence)
740
+ y_pred_train = best_model.predict(X_train)
741
+ if is_clf:
742
+ train_score = accuracy_score(y_train, y_pred_train)
743
+ test_score = accuracy_score(y_test, best_result['y_pred'])
744
+ else:
745
+ train_score = r2_score(y_train, y_pred_train)
746
+ test_score = r2_score(y_test, best_result['y_pred'])
747
+ gap = train_score - test_score
748
+ if gap > 0.1:
749
+ print(f"\n⚠️ Overfitting detected (train-test gap: {gap:.4f})")
750
+ elif gap < 0.02:
751
+ print(f"\n✅ Good generalization (train-test gap: {gap:.4f})")
752
+
753
+ return best_model, best_name, best_result['metrics'], results
754
+
755
+
756
+ def main():
757
+ parser = argparse.ArgumentParser(description="Train ML model (reproduces your DataVision AutoML)")
758
+ parser.add_argument('--data', type=str, default='data/cleaned_data.csv', help='Path to CSV data file')
759
+ parser.add_argument('--output', type=str, default='model.pkl', help='Output model file')
760
+ parser.add_argument('--test-size', type=float, default=0.2, help='Test split ratio (DataVision uses 0.2)')
761
+ parser.add_argument('--all', action='store_true', help='Train ALL algorithms (like full AutoML)')
762
+ parser.add_argument('--mode', type=str, default='fast', choices=['fast', 'ultra'],
763
+ help='Training mode when using --all (fast: ~10 models, ultra: ~25 models)')
764
+ args = parser.parse_args()
765
+
766
+ print("\n" + "=" * 70)
767
+ if args.all:
768
+ print("🚀 DataVision AutoML — Training ALL Algorithms")
769
+ else:
770
+ print(f"🚀 DataVision AutoML — Training YOUR Winning Model: RandomForest")
771
+ print(" (Use --all to train all algorithms like full AutoML)")
772
+ print("=" * 70)
773
+
774
+ if not os.path.exists(args.data):
775
+ print(f"❌ Data file not found: {args.data}")
776
+ return
777
+
778
+ start_time = time.time()
779
+
780
+ # Load data
781
+ df = pd.read_csv(args.data)
782
+
783
+ # Step 1: Clean data (ProductionDataCleaner)
784
+ df = clean_data(df)
785
+ df = df.dropna(subset=[TARGET_COLUMN])
786
+
787
+ # Separate target BEFORE feature engineering (anti-leakage — same as DataVision)
788
+ X = df.drop(columns=[TARGET_COLUMN])
789
+ y = df[TARGET_COLUMN]
790
+
791
+ # Encode target
792
+ target_encoder = None
793
+ if TASK_TYPE == 'classification':
794
+ target_encoder = LabelEncoder()
795
+ y = pd.Series(target_encoder.fit_transform(y), name=TARGET_COLUMN)
796
+ print(f"\n🎯 Target: price_range ({len(target_encoder.classes_)} classes: {list(target_encoder.classes_)[:10]})")
797
+ else:
798
+ print(f"\n🎯 Target: price_range (regression)")
799
+
800
+ # Train/test split BEFORE feature engineering (anti-leakage)
801
+ X_train, X_test, y_train, y_test = train_test_split(
802
+ X, y, test_size=args.test_size, random_state=42,
803
+ stratify=y if TASK_TYPE == 'classification' else None
804
+ )
805
+ print(f"📦 Split: {len(X_train)} train / {len(X_test)} test")
806
+ print(f" ⚠️ Feature engineering applied on train ONLY (anti-leakage)")
807
+
808
+ # Step 2: Feature engineering (fit on train, transform test — anti-leakage)
809
+ X_train_eng, label_encoders, scaler = engineer_features(X_train.copy(), mode=args.mode)
810
+
811
+ # Transform test set using same encoding (simplified — uses same column structure)
812
+ X_test_eng = X_test.copy()
813
+ # Apply same encoding to test
814
+ for col in X_test_eng.select_dtypes(include=['object', 'category']).columns:
815
+ if col in label_encoders:
816
+ le = label_encoders[col]
817
+ X_test_eng[col] = X_test_eng[col].astype(str).map(
818
+ lambda x: le.transform([x])[0] if x in le.classes_ else 0
819
+ )
820
+ else:
821
+ le = LabelEncoder()
822
+ X_test_eng[col] = le.fit_transform(X_test_eng[col].astype(str))
823
+ for col in X_test_eng.select_dtypes(include=['object', 'category']).columns:
824
+ X_test_eng[col] = pd.to_numeric(X_test_eng[col], errors='coerce').fillna(0)
825
+ X_test_eng = X_test_eng.fillna(0)
826
+ X_test_eng = X_test_eng.replace([np.inf, -np.inf], 0)
827
+
828
+ # Ensure same columns as train
829
+ for col in X_train_eng.columns:
830
+ if col not in X_test_eng.columns:
831
+ X_test_eng[col] = 0
832
+ X_test_eng = X_test_eng[X_train_eng.columns]
833
+ for col in X_test_eng.columns:
834
+ X_test_eng[col] = pd.to_numeric(X_test_eng[col], errors='coerce').fillna(0)
835
+
836
+ # Step 3 & 4: Train models
837
+ # DEFAULT: Train ONLY your winning model (to reproduce exact result)
838
+ # --all: Train all algorithms like full AutoML
839
+ if args.all:
840
+ print(f"\n📊 Training ALL models (mode={args.mode})...")
841
+ best_model, best_name, metrics, all_results = train_and_evaluate(
842
+ X_train_eng, X_test_eng, y_train, y_test, mode=args.mode
843
+ )
844
+ else:
845
+ print(f"\n📊 Training ONLY your winning model: RandomForest")
846
+ models = get_single_model(BEST_MODEL_NAME, len(X_train_eng))
847
+ best_model, best_name, metrics, all_results = train_and_evaluate(
848
+ X_train_eng, X_test_eng, y_train, y_test, mode=args.mode, selected_models=models
849
+ )
850
+
851
+ if best_model is None:
852
+ print("\n⚠️ Selected model failed. Retrying with ALL algorithms...")
853
+ best_model, best_name, metrics, all_results = train_and_evaluate(
854
+ X_train_eng, X_test_eng, y_train, y_test, mode='fast'
855
+ )
856
+
857
+ if best_model is None:
858
+ print("\n❌ All models failed. Check your data and dependencies.")
859
+ return
860
+
861
+ total_time = time.time() - start_time
862
+
863
+ # Save model state (same structure as DataVision's engine_state)
864
+ engine_state = {
865
+ 'model': best_model,
866
+ 'model_name': best_name,
867
+ 'task_type': TASK_TYPE,
868
+ 'task_type_simple': TASK_TYPE,
869
+ 'target_column': TARGET_COLUMN,
870
+ 'feature_columns': X_train_eng.columns.tolist(),
871
+ 'label_encoders': label_encoders,
872
+ 'target_encoder': target_encoder,
873
+ 'scaler': scaler,
874
+ 'metrics': metrics,
875
+ 'production_mode': True,
876
+ 'mode': args.mode,
877
+ 'training_time_seconds': total_time,
878
+ 'leaderboard': [
879
+ {'name': r['name'], 'score': r['score'], 'metrics': r['metrics'], 'time': r.get('time', 0)}
880
+ for r in all_results
881
+ ],
882
+ }
883
+
884
+ with open(args.output, 'wb') as f:
885
+ pickle.dump(engine_state, f)
886
+
887
+ file_size = os.path.getsize(args.output) / 1024
888
+ print(f"\n💾 Model saved to: {args.output} ({file_size:.1f} KB)")
889
+ print(f"⏱️ Total time: {total_time:.1f}s")
890
+ print("✅ Training complete!")
891
+
892
+
893
+ if __name__ == '__main__':
894
+ main()
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/utils/__init__.py ADDED
File without changes
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/utils/preprocessing.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🔧 Preprocessing Utilities — Mirrors DataVision AutoML Pipeline
3
+ ===================================================================
4
+ These utilities replicate the EXACT preprocessing pipeline used by
5
+ DataVision's ProductionDataCleaner + ProductionFeatureEngineer.
6
+
7
+ Pipeline steps:
8
+ 1. Remove useless columns (IDs, URLs, high-cardinality strings)
9
+ 2. Remove duplicates
10
+ 3. Smart type conversion (parse $, M/K units, commas)
11
+ 4. KNN imputation for numerics (median fallback)
12
+ 5. Mode fill for categoricals
13
+ 6. Replace inf/-inf with median
14
+ 7. Remove constant columns
15
+ 8. IQR-based outlier capping
16
+ 9. Log-transform skewed features
17
+ 10. RobustScaler for numeric features
18
+ 11. Categorical encoding (binary→label, low-card→one-hot, high-card→label+freq)
19
+ 12. VarianceThreshold to remove near-constant features
20
+ """
21
+
22
+ import numpy as np
23
+ import pandas as pd
24
+ from sklearn.preprocessing import LabelEncoder, RobustScaler
25
+ from sklearn.feature_selection import VarianceThreshold
26
+ from sklearn.impute import KNNImputer
27
+
28
+ TARGET_COLUMN = 'price_range'
29
+ TASK_TYPE = 'classification'
30
+ FEATURE_COLUMNS = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt', 'n_cores', 'pc', 'px_height', 'px_width', 'ram', 'sc_h', 'sc_w', 'talk_time', 'three_g', 'touch_screen', 'wifi']
31
+
32
+
33
+ def detect_task_type(y):
34
+ """Auto-detect classification vs regression."""
35
+ if y.dtype == 'object' or y.nunique() < 20:
36
+ return 'classification'
37
+ return 'regression'
38
+
39
+
40
+ def remove_useless_columns(df, target_col=TARGET_COLUMN):
41
+ """Remove ID-like, URL, and high-cardinality columns (same as DataVision)."""
42
+ id_patterns = ['unnamed', 'index', '_id', 'guid', 'uuid', 'url', 'link', 'href', 'path']
43
+ cols_to_drop = []
44
+ for col in df.columns:
45
+ if col == target_col:
46
+ continue
47
+ col_lower = col.lower().strip()
48
+ if any(pat in col_lower for pat in id_patterns):
49
+ cols_to_drop.append(col)
50
+ elif df[col].dtype == 'object' and df[col].nunique() / max(len(df), 1) > 0.9:
51
+ cols_to_drop.append(col)
52
+ return df.drop(columns=cols_to_drop, errors='ignore') if cols_to_drop else df
53
+
54
+
55
+ def smart_type_conversion(df, target_col=TARGET_COLUMN):
56
+ """Parse hidden numerics: $, M/K units, commas (same as DataVision)."""
57
+ for col in df.select_dtypes(include=['object']).columns:
58
+ if col == target_col:
59
+ continue
60
+ sample = df[col].dropna().head(20)
61
+ if len(sample) == 0:
62
+ continue
63
+ converted = 0
64
+ for val in sample:
65
+ try:
66
+ v = str(val).replace('$', '').replace(',', '').strip()
67
+ if v.upper().endswith(('M', 'K', '+')):
68
+ float(v[:-1])
69
+ else:
70
+ float(v)
71
+ converted += 1
72
+ except:
73
+ pass
74
+ if converted / max(len(sample), 1) > 0.6:
75
+ def parse_val(val):
76
+ if pd.isna(val):
77
+ return np.nan
78
+ v = str(val).replace('$', '').replace(',', '').strip()
79
+ try:
80
+ if v.upper().endswith('M'): return float(v[:-1]) * 1e6
81
+ if v.upper().endswith('K'): return float(v[:-1]) * 1e3
82
+ if v.endswith('+'): return float(v[:-1])
83
+ return float(v)
84
+ except:
85
+ return np.nan
86
+ df[col] = df[col].apply(parse_val)
87
+ return df
88
+
89
+
90
+ def impute_missing(df, target_col=TARGET_COLUMN):
91
+ """KNN imputation for numerics, mode fill for categoricals (same as DataVision)."""
92
+ numeric_cols = [c for c in df.select_dtypes(include=[np.number]).columns if c != target_col]
93
+ if numeric_cols and df[numeric_cols].isna().any().any():
94
+ try:
95
+ imputer = KNNImputer(n_neighbors=5, weights='distance')
96
+ df[numeric_cols] = imputer.fit_transform(df[numeric_cols])
97
+ except:
98
+ for col in numeric_cols:
99
+ df[col] = df[col].fillna(df[col].median())
100
+
101
+ for col in df.select_dtypes(include=['object', 'category']).columns:
102
+ if col == target_col:
103
+ continue
104
+ mode = df[col].mode()
105
+ df[col] = df[col].fillna(mode.iloc[0] if len(mode) > 0 else '_MISSING_')
106
+ return df
107
+
108
+
109
+ def cap_outliers_iqr(df, target_col=TARGET_COLUMN):
110
+ """IQR-based outlier capping with [1st, 99th] percentile bounds (same as DataVision)."""
111
+ for col in df.select_dtypes(include=[np.number]).columns:
112
+ if col == target_col:
113
+ continue
114
+ Q1, Q3 = df[col].quantile(0.25), df[col].quantile(0.75)
115
+ IQR = Q3 - Q1
116
+ p1, p99 = df[col].quantile(0.01), df[col].quantile(0.99)
117
+ lower = max(p1, Q1 - 3 * IQR)
118
+ upper = min(p99, Q3 + 3 * IQR)
119
+ df[col] = df[col].clip(lower=lower, upper=upper)
120
+ return df
121
+
122
+
123
+ def engineer_features(X):
124
+ """Feature engineering matching DataVision's ProductionFeatureEngineer."""
125
+ label_encoders = {}
126
+ numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
127
+ categorical_cols = X.select_dtypes(include=['object', 'category']).columns.tolist()
128
+
129
+ # Log-transform skewed numerics (skewness > 2, non-negative)
130
+ for col in numeric_cols:
131
+ if (X[col] >= 0).all() and abs(X[col].skew()) > 2:
132
+ X[f'{col}_log'] = np.log1p(X[col])
133
+
134
+ # RobustScaler (same as DataVision, not StandardScaler)
135
+ numeric_cols_all = X.select_dtypes(include=[np.number]).columns.tolist()
136
+ scaler = RobustScaler()
137
+ if numeric_cols_all:
138
+ X[numeric_cols_all] = scaler.fit_transform(X[numeric_cols_all])
139
+
140
+ # Polynomial features (top 4 numeric, squared + interactions)
141
+ if 2 <= len(numeric_cols) <= 10:
142
+ top_n = min(4, len(numeric_cols))
143
+ top_cols = numeric_cols[:top_n]
144
+ for col in top_cols:
145
+ X[f'{col}_sq'] = X[col] ** 2
146
+ for i in range(len(top_cols)):
147
+ for j in range(i + 1, len(top_cols)):
148
+ X[f'{top_cols[i]}_x_{top_cols[j]}'] = X[top_cols[i]] * X[top_cols[j]]
149
+
150
+ # Categorical encoding
151
+ for col in categorical_cols:
152
+ nuniq = X[col].nunique()
153
+ if nuniq <= 2: # Binary → LabelEncoder
154
+ le = LabelEncoder()
155
+ X[col] = le.fit_transform(X[col].astype(str))
156
+ label_encoders[col] = le
157
+ elif nuniq <= 10: # Low cardinality → one-hot
158
+ dummies = pd.get_dummies(X[col], prefix=col, drop_first=False)
159
+ X = pd.concat([X.drop(columns=[col]), dummies], axis=1)
160
+ else: # High cardinality → label + frequency encoding
161
+ le = LabelEncoder()
162
+ X[f'{col}_label'] = le.fit_transform(X[col].astype(str))
163
+ freq = X[col].value_counts(normalize=True)
164
+ X[f'{col}_freq'] = X[col].map(freq)
165
+ label_encoders[col] = le
166
+ X = X.drop(columns=[col])
167
+
168
+ # Force float + clean
169
+ for col in X.columns:
170
+ try:
171
+ X[col] = X[col].astype(float)
172
+ except:
173
+ le = LabelEncoder()
174
+ X[col] = le.fit_transform(X[col].astype(str)).astype(float)
175
+ X = X.replace([np.inf, -np.inf], 0).fillna(0)
176
+
177
+ # VarianceThreshold (remove near-constant, threshold=0.01)
178
+ try:
179
+ vt = VarianceThreshold(threshold=0.01)
180
+ X = pd.DataFrame(vt.fit_transform(X), columns=X.columns[vt.get_support()], index=X.index)
181
+ except:
182
+ pass
183
+
184
+ return X, label_encoders, scaler
185
+
186
+
187
+ def full_preprocess(df, target_column=TARGET_COLUMN):
188
+ """Complete DataVision preprocessing pipeline.
189
+
190
+ Combines ProductionDataCleaner.clean() + ProductionFeatureEngineer.fit_transform().
191
+ """
192
+ # Step 1: Clean
193
+ df = df.dropna(subset=[target_column])
194
+ df = df.drop_duplicates()
195
+
196
+ # Strip whitespace and replace string NaN variants
197
+ for col in df.select_dtypes(include=['object']).columns:
198
+ df[col] = df[col].str.strip()
199
+ df[col] = df[col].replace(['nan', 'NaN', 'None', 'null', 'NULL', ''], np.nan)
200
+
201
+ df = remove_useless_columns(df, target_column)
202
+ df = smart_type_conversion(df, target_column)
203
+ df = impute_missing(df, target_column)
204
+
205
+ # Replace inf, remove constant columns
206
+ for col in df.select_dtypes(include=[np.number]).columns:
207
+ if col != target_column:
208
+ df[col] = df[col].replace([np.inf, -np.inf], df[col].replace([np.inf, -np.inf], np.nan).median())
209
+ const_cols = [c for c in df.columns if c != target_column and df[c].nunique() <= 1]
210
+ df = df.drop(columns=const_cols, errors='ignore')
211
+
212
+ df = cap_outliers_iqr(df, target_column)
213
+ df[df.select_dtypes(include=[np.number]).columns] = df.select_dtypes(include=[np.number]).fillna(0)
214
+
215
+ # Step 2: Split target
216
+ X = df.drop(columns=[target_column])
217
+ y = df[target_column]
218
+
219
+ task_type = detect_task_type(y)
220
+ target_encoder = None
221
+ if task_type == 'classification':
222
+ target_encoder = LabelEncoder()
223
+ y = pd.Series(target_encoder.fit_transform(y))
224
+
225
+ # Step 3: Engineer features
226
+ X, label_encoders, scaler = engineer_features(X)
227
+ feature_cols = X.columns.tolist()
228
+
229
+ return X, y, task_type, target_encoder, label_encoders, scaler, feature_cols
backend/.workspace/87b47e72-5950-42e0-9196-791c791ff5ee/visualize.py ADDED
@@ -0,0 +1,944 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 📊 Visualization Script — ALL Charts Matching DataVision AutoML
3
+ ================================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ Model : RandomForest
7
+ Target Column : price_range
8
+ Task Type : classification
9
+ Features : 20 columns
10
+
11
+ Supports Traditional ML AND Deep Learning models.
12
+ Automatically loads the correct model based on BEST_MODE.
13
+
14
+ Generates the SAME charts you see in DataVision AutoML:
15
+ Classification: confusion matrix, class distribution, ROC curve,
16
+ precision-recall, calibration, confidence histogram,
17
+ metrics heatmap, feature importance, model comparison,
18
+ correlation heatmap, feature distributions, boxplots
19
+ Regression: actual vs predicted, residuals analysis, error distribution,
20
+ prediction overview, feature importance, model comparison,
21
+ correlation heatmap, feature distributions, boxplots
22
+ Deep Learning: training loss curve, validation score curve,
23
+ model architecture summary, MLP weight importance
24
+ Unsupervised: KMeans clustering scatter plot (PCA 2D),
25
+ cluster distribution chart
26
+
27
+ Usage:
28
+ python visualize.py # Auto-detect best model
29
+ python visualize.py --mode dl # Force Deep Learning model
30
+ python visualize.py --mode traditional # Force traditional ML model
31
+ python visualize.py --data my_data.csv
32
+ python visualize.py --output charts/
33
+ """
34
+
35
+ import os
36
+ import pickle
37
+ import argparse
38
+ import warnings
39
+ import numpy as np
40
+ import pandas as pd
41
+ import matplotlib
42
+ matplotlib.use('Agg')
43
+ import matplotlib.pyplot as plt
44
+ import seaborn as sns
45
+ from sklearn.model_selection import train_test_split
46
+ from sklearn.preprocessing import LabelEncoder, label_binarize
47
+ from sklearn.metrics import (
48
+ accuracy_score, confusion_matrix, classification_report,
49
+ r2_score, mean_absolute_error, mean_squared_error,
50
+ roc_curve, auc, precision_recall_curve, average_precision_score,
51
+ precision_score, recall_score, f1_score
52
+ )
53
+
54
+ try:
55
+ from sklearn.calibration import calibration_curve
56
+ HAS_CALIBRATION = True
57
+ except ImportError:
58
+ HAS_CALIBRATION = False
59
+
60
+ warnings.filterwarnings('ignore')
61
+ sns.set_style("whitegrid")
62
+ plt.rcParams.update({
63
+ 'figure.figsize': (10, 6),
64
+ 'font.size': 12,
65
+ 'axes.titlesize': 14,
66
+ 'axes.labelsize': 12,
67
+ })
68
+
69
+ TARGET_COLUMN = 'price_range'
70
+ TASK_TYPE = 'classification'
71
+ BEST_MODE = 'traditional' # Which mode won during AutoML
72
+ NUMERIC_FEATURES = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt']
73
+ LEADERBOARD_NAMES = ['?']
74
+ LEADERBOARD_SCORES = [0.8925]
75
+
76
+
77
+ def load_model_and_data(model_path='model.pkl', data_path='data/cleaned_data.csv'):
78
+ if not os.path.exists(model_path):
79
+ print(f"❌ Model not found: {model_path}")
80
+ return None, None
81
+ with open(model_path, 'rb') as f:
82
+ state = pickle.load(f)
83
+ df = pd.read_csv(data_path) if os.path.exists(data_path) else None
84
+ return state, df
85
+
86
+
87
+ def load_dl_model(dl_path='deep_learning_model.pkl'):
88
+ """Load the Deep Learning model (sklearn MLP)."""
89
+ if not os.path.exists(dl_path):
90
+ return None
91
+ try:
92
+ with open(dl_path, 'rb') as f:
93
+ state = pickle.load(f)
94
+ print(f"✅ Loaded DL model: {state.get('algorithm', 'MLP')}")
95
+ print(f" Task: {state.get('task_type', 'Unknown')}")
96
+ print(f" Features: {len(state.get('feature_columns', state.get('feature_metadata', [])))}")
97
+ return state
98
+ except Exception as e:
99
+ print(f"⚠️ Failed to load DL model: {e}")
100
+ return None
101
+
102
+
103
+ def generate_charts(state, df, output_dir='charts'):
104
+ os.makedirs(output_dir, exist_ok=True)
105
+
106
+ model = state['model']
107
+ task_type = state.get('task_type', 'classification')
108
+ target_col = state.get('target_column', TARGET_COLUMN)
109
+ feature_cols = state.get('feature_columns', [])
110
+ is_clf = 'classif' in task_type.lower()
111
+ is_dl = state.get('model_type') == 'deep_learning'
112
+ charts = []
113
+
114
+ # Load SAVED predictions from AutoML (exact results, no feature mismatch)
115
+ y_test = np.array(state['y_test']) if state.get('y_test') is not None else None
116
+ y_pred = np.array(state['y_pred']) if state.get('y_pred') is not None else None
117
+ y_proba = np.array(state['y_proba']) if state.get('y_proba') is not None else None
118
+ # Handle both ML (target_encoder) and DL (label_encoder) naming
119
+ target_encoder = state.get('target_encoder', None)
120
+ label_encoder = state.get('label_encoder', None)
121
+ encoder = target_encoder or label_encoder
122
+ class_names = list(encoder.classes_) if encoder is not None else None
123
+ mode_display = 'Deep Learning (MLP)' if is_dl else 'Traditional ML'
124
+ model_name = state.get('model_name', state.get('algorithm', 'Unknown'))
125
+
126
+ print(f"\n📊 Generating charts for {model_name} ({mode_display})...")
127
+ print(f" Saved predictions: y_test={y_test is not None}, y_pred={y_pred is not None}, y_proba={y_proba is not None}")
128
+
129
+ # =====================================================================
130
+ # 1. MODEL COMPARISON / LEADERBOARD (same as AutoML)
131
+ # =====================================================================
132
+ if LEADERBOARD_NAMES and LEADERBOARD_SCORES:
133
+ try:
134
+ fig, ax = plt.subplots(figsize=(10, max(4, len(LEADERBOARD_NAMES) * 0.5)))
135
+ colors = sns.color_palette('viridis', len(LEADERBOARD_NAMES))
136
+ y_pos = range(len(LEADERBOARD_NAMES))
137
+ ax.barh(y_pos, LEADERBOARD_SCORES, color=colors)
138
+ ax.set_yticks(y_pos)
139
+ ax.set_yticklabels(LEADERBOARD_NAMES)
140
+ metric_label = 'F1-Score' if is_clf else 'R²'
141
+ ax.set_xlabel(metric_label)
142
+ ax.set_title(f'Model Leaderboard — {metric_label}')
143
+ for i, v in enumerate(LEADERBOARD_SCORES):
144
+ ax.text(v + 0.001, i, f'{v:.4f}', va='center', fontsize=9)
145
+ plt.tight_layout()
146
+ path = os.path.join(output_dir, 'model_comparison.png')
147
+ plt.savefig(path, dpi=150, bbox_inches='tight')
148
+ plt.close()
149
+ charts.append(path)
150
+ print(f" ✅ Model Comparison → {path}")
151
+ except Exception as e:
152
+ print(f" ⚠️ Model Comparison failed: {e}")
153
+
154
+ # =====================================================================
155
+ # 2. FEATURE IMPORTANCE (from model)
156
+ # =====================================================================
157
+ try:
158
+ importances = None
159
+ if hasattr(model, 'feature_importances_'):
160
+ # Tree-based models (RF, XGB, etc.)
161
+ importances = model.feature_importances_
162
+ elif hasattr(model, 'coefs_') and len(model.coefs_) > 0:
163
+ # MLP/Neural Network — use mean absolute weight of first layer
164
+ importances = np.mean(np.abs(model.coefs_[0]), axis=1)
165
+ elif hasattr(model, 'coef_'):
166
+ # Linear models (LogReg, SVM, etc.)
167
+ coef = model.coef_
168
+ importances = np.mean(np.abs(coef), axis=0) if coef.ndim > 1 else np.abs(coef)
169
+
170
+ if importances is not None:
171
+ names = feature_cols[:len(importances)] if feature_cols else [f'Feature_{i}' for i in range(len(importances))]
172
+ fi = pd.Series(importances, index=names).nlargest(min(20, len(names)))
173
+
174
+ fig, ax = plt.subplots(figsize=(10, max(4, len(fi) * 0.35)))
175
+ bars = fi.sort_values().plot(kind='barh', ax=ax,
176
+ color=plt.cm.viridis(np.linspace(0.2, 0.8, len(fi))))
177
+ title_suffix = ' (MLP Weight Magnitude)' if is_dl else ''
178
+ ax.set_title(f'Feature Importance — Top Features{title_suffix}')
179
+ ax.set_xlabel('Importance')
180
+ for i, (val, name) in enumerate(zip(fi.sort_values(), fi.sort_values().index)):
181
+ ax.text(val + 0.001, i, f'{val:.4f}', va='center', fontsize=8)
182
+ plt.tight_layout()
183
+ path = os.path.join(output_dir, 'feature_importance.png')
184
+ plt.savefig(path, dpi=150, bbox_inches='tight')
185
+ plt.close()
186
+ charts.append(path)
187
+ print(f" ✅ Feature Importance → {path}")
188
+ except Exception as e:
189
+ print(f" ⚠️ Feature Importance failed: {e}")
190
+
191
+ # =====================================================================
192
+ # DL-SPECIFIC: TRAINING HISTORY (loss curve + validation score)
193
+ # =====================================================================
194
+ if is_dl:
195
+ training_history = state.get('training_history', {})
196
+ loss_curve = training_history.get('loss', [])
197
+ val_scores = training_history.get('val_score', [])
198
+
199
+ if loss_curve:
200
+ try:
201
+ n_plots = 1 + (1 if val_scores else 0)
202
+ fig, axes = plt.subplots(1, n_plots, figsize=(6 * n_plots, 5))
203
+ if n_plots == 1:
204
+ axes = [axes]
205
+
206
+ # Loss curve
207
+ axes[0].plot(range(1, len(loss_curve) + 1), loss_curve, color='#EF4444', lw=2)
208
+ axes[0].set_xlabel('Epoch')
209
+ axes[0].set_ylabel('Loss')
210
+ axes[0].set_title('Training Loss Curve')
211
+ axes[0].grid(True, alpha=0.3)
212
+
213
+ # Validation score curve
214
+ if val_scores and n_plots > 1:
215
+ axes[1].plot(range(1, len(val_scores) + 1), val_scores, color='#22C55E', lw=2)
216
+ axes[1].set_xlabel('Epoch')
217
+ axes[1].set_ylabel('Validation Score')
218
+ axes[1].set_title('Validation Score Curve')
219
+ axes[1].grid(True, alpha=0.3)
220
+
221
+ plt.suptitle(f'Deep Learning Training History — {state.get("algorithm", "MLP")}', fontsize=14)
222
+ plt.tight_layout()
223
+ path = os.path.join(output_dir, 'training_history.png')
224
+ plt.savefig(path, dpi=150, bbox_inches='tight')
225
+ plt.close()
226
+ charts.append(path)
227
+ print(f" ✅ Training History → {path}")
228
+ except Exception as e:
229
+ print(f" ⚠️ Training History failed: {e}")
230
+
231
+ # DL Model Architecture Summary
232
+ try:
233
+ fig, ax = plt.subplots(figsize=(8, 4))
234
+ ax.axis('off')
235
+ info_lines = [
236
+ f"Model: {state.get('algorithm', 'MLP')}",
237
+ f"Task: {state.get('task_type', 'Unknown')}",
238
+ f"Features: {len(feature_cols)}",
239
+ ]
240
+ if hasattr(model, 'hidden_layer_sizes'):
241
+ info_lines.append(f"Hidden Layers: {model.hidden_layer_sizes}")
242
+ if hasattr(model, 'activation'):
243
+ info_lines.append(f"Activation: {model.activation}")
244
+ if hasattr(model, 'n_iter_'):
245
+ info_lines.append(f"Epochs Trained: {model.n_iter_}")
246
+ metrics = state.get('metrics', {})
247
+ if metrics:
248
+ info_lines.append("")
249
+ info_lines.append("Metrics:")
250
+ for k, v in metrics.items():
251
+ if isinstance(v, float):
252
+ info_lines.append(f" {k}: {v:.4f}")
253
+ text = "\n".join(info_lines)
254
+ ax.text(0.1, 0.9, text, transform=ax.transAxes, fontsize=12,
255
+ verticalalignment='top', fontfamily='monospace',
256
+ bbox=dict(boxstyle='round', facecolor='#F0F9FF', alpha=0.8))
257
+ ax.set_title('Deep Learning Model Summary', fontsize=14, fontweight='bold')
258
+ plt.tight_layout()
259
+ path = os.path.join(output_dir, 'dl_model_summary.png')
260
+ plt.savefig(path, dpi=150, bbox_inches='tight')
261
+ plt.close()
262
+ charts.append(path)
263
+ print(f" ✅ DL Model Summary → {path}")
264
+ except Exception as e:
265
+ print(f" ⚠️ DL Model Summary failed: {e}")
266
+
267
+ # =====================================================================
268
+ # CLASSIFICATION-SPECIFIC CHARTS
269
+ # =====================================================================
270
+ if is_clf and y_test is not None and y_pred is not None:
271
+ n_classes = len(np.unique(y_test))
272
+ is_binary = n_classes == 2
273
+
274
+ # --- 3. CONFUSION MATRIX (normalized + counts) ---
275
+ try:
276
+ cm = confusion_matrix(y_test, y_pred)
277
+ cm_norm = cm.astype('float') / cm.sum(axis=1, keepdims=True)
278
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
279
+
280
+ fig, axes = plt.subplots(1, 2, figsize=(14, 5))
281
+ # Counts
282
+ sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
283
+ xticklabels=labels, yticklabels=labels)
284
+ axes[0].set_title('Confusion Matrix (Counts)')
285
+ axes[0].set_xlabel('Predicted')
286
+ axes[0].set_ylabel('Actual')
287
+ # Normalized
288
+ sns.heatmap(cm_norm, annot=True, fmt='.2%', cmap='Blues', ax=axes[1],
289
+ xticklabels=labels, yticklabels=labels)
290
+ axes[1].set_title('Confusion Matrix (Normalized)')
291
+ axes[1].set_xlabel('Predicted')
292
+ axes[1].set_ylabel('Actual')
293
+ plt.tight_layout()
294
+ path = os.path.join(output_dir, 'confusion_matrix.png')
295
+ plt.savefig(path, dpi=150, bbox_inches='tight')
296
+ plt.close()
297
+ charts.append(path)
298
+ print(f" ✅ Confusion Matrix → {path}")
299
+ except Exception as e:
300
+ print(f" ⚠️ Confusion Matrix failed: {e}")
301
+
302
+ # --- 4. CLASS DISTRIBUTION (actual vs predicted) ---
303
+ try:
304
+ fig, axes = plt.subplots(1, 2, figsize=(12, 5))
305
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
306
+ # Actual
307
+ actual_counts = pd.Series(y_test).value_counts().sort_index()
308
+ pred_counts = pd.Series(y_pred).value_counts().sort_index()
309
+ axes[0].bar(range(len(actual_counts)), actual_counts.values, color=sns.color_palette('Set2'))
310
+ axes[0].set_xticks(range(len(actual_counts)))
311
+ axes[0].set_xticklabels(labels, rotation=45, ha='right')
312
+ axes[0].set_title('Actual Distribution')
313
+ axes[0].set_ylabel('Count')
314
+ axes[1].bar(range(len(pred_counts)), pred_counts.reindex(actual_counts.index, fill_value=0).values,
315
+ color=sns.color_palette('Set3'))
316
+ axes[1].set_xticks(range(len(actual_counts)))
317
+ axes[1].set_xticklabels(labels, rotation=45, ha='right')
318
+ axes[1].set_title('Predicted Distribution')
319
+ axes[1].set_ylabel('Count')
320
+ plt.suptitle('Class Distribution: Actual vs Predicted', fontsize=14)
321
+ plt.tight_layout()
322
+ path = os.path.join(output_dir, 'class_distribution.png')
323
+ plt.savefig(path, dpi=150, bbox_inches='tight')
324
+ plt.close()
325
+ charts.append(path)
326
+ print(f" ✅ Class Distribution → {path}")
327
+ except Exception as e:
328
+ print(f" ⚠️ Class Distribution failed: {e}")
329
+
330
+ # --- 5. ROC CURVE (binary + multi-class) ---
331
+ if y_proba is not None:
332
+ try:
333
+ fig, ax = plt.subplots(figsize=(8, 8))
334
+ if is_binary:
335
+ proba_pos = y_proba[:, 1] if y_proba.ndim == 2 else y_proba
336
+ fpr, tpr, _ = roc_curve(y_test, proba_pos)
337
+ roc_auc = auc(fpr, tpr)
338
+ ax.plot(fpr, tpr, color='#4F46E5', lw=2, label=f'ROC (AUC = {roc_auc:.4f})')
339
+ else:
340
+ y_test_bin = label_binarize(y_test, classes=sorted(np.unique(y_test)))
341
+ colors = plt.cm.Set1(np.linspace(0, 1, n_classes))
342
+ for i in range(min(n_classes, y_proba.shape[1])):
343
+ fpr, tpr, _ = roc_curve(y_test_bin[:, i], y_proba[:, i])
344
+ roc_auc = auc(fpr, tpr)
345
+ label = class_names[i] if class_names and i < len(class_names) else f'Class {i}'
346
+ ax.plot(fpr, tpr, color=colors[i], lw=2, label=f'{label} (AUC={roc_auc:.3f})')
347
+ ax.plot([0, 1], [0, 1], 'k--', lw=1, alpha=0.5)
348
+ ax.set_xlabel('False Positive Rate')
349
+ ax.set_ylabel('True Positive Rate')
350
+ ax.set_title('ROC Curve')
351
+ ax.legend(loc='lower right')
352
+ ax.set_xlim([0, 1])
353
+ ax.set_ylim([0, 1.02])
354
+ plt.tight_layout()
355
+ path = os.path.join(output_dir, 'roc_curve.png')
356
+ plt.savefig(path, dpi=150, bbox_inches='tight')
357
+ plt.close()
358
+ charts.append(path)
359
+ print(f" ✅ ROC Curve → {path}")
360
+ except Exception as e:
361
+ print(f" ⚠️ ROC Curve failed: {e}")
362
+
363
+ # --- 6. PRECISION-RECALL CURVE (binary) ---
364
+ if y_proba is not None and is_binary:
365
+ try:
366
+ proba_pos = y_proba[:, 1] if y_proba.ndim == 2 else y_proba
367
+ precision_vals, recall_vals, _ = precision_recall_curve(y_test, proba_pos)
368
+ ap = average_precision_score(y_test, proba_pos)
369
+ fig, ax = plt.subplots(figsize=(8, 6))
370
+ ax.plot(recall_vals, precision_vals, color='#059669', lw=2,
371
+ label=f'AP = {ap:.4f}')
372
+ ax.set_xlabel('Recall')
373
+ ax.set_ylabel('Precision')
374
+ ax.set_title('Precision-Recall Curve')
375
+ ax.legend(loc='lower left')
376
+ ax.set_xlim([0, 1])
377
+ ax.set_ylim([0, 1.02])
378
+ plt.tight_layout()
379
+ path = os.path.join(output_dir, 'precision_recall.png')
380
+ plt.savefig(path, dpi=150, bbox_inches='tight')
381
+ plt.close()
382
+ charts.append(path)
383
+ print(f" ✅ Precision-Recall → {path}")
384
+ except Exception as e:
385
+ print(f" ⚠️ Precision-Recall failed: {e}")
386
+
387
+ # --- 7. CALIBRATION CURVE (binary) ---
388
+ if y_proba is not None and is_binary and HAS_CALIBRATION and len(y_test) >= 20:
389
+ try:
390
+ proba_pos = y_proba[:, 1] if y_proba.ndim == 2 else y_proba
391
+ fraction_of_positives, mean_predicted_value = calibration_curve(
392
+ y_test, proba_pos, n_bins=10
393
+ )
394
+ fig, ax = plt.subplots(figsize=(8, 6))
395
+ ax.plot(mean_predicted_value, fraction_of_positives, 's-', color='#4F46E5',
396
+ label='Model', lw=2)
397
+ ax.plot([0, 1], [0, 1], 'k--', label='Perfectly calibrated')
398
+ ax.set_xlabel('Mean Predicted Probability')
399
+ ax.set_ylabel('Fraction of Positives')
400
+ ax.set_title('Calibration Curve')
401
+ ax.legend()
402
+ plt.tight_layout()
403
+ path = os.path.join(output_dir, 'calibration.png')
404
+ plt.savefig(path, dpi=150, bbox_inches='tight')
405
+ plt.close()
406
+ charts.append(path)
407
+ print(f" ✅ Calibration Curve → {path}")
408
+ except Exception as e:
409
+ print(f" ⚠️ Calibration Curve failed: {e}")
410
+
411
+ # --- 8. CONFIDENCE HISTOGRAM ---
412
+ if y_proba is not None:
413
+ try:
414
+ max_proba = np.max(y_proba, axis=1) if y_proba.ndim == 2 else y_proba
415
+ correct = (y_test == y_pred)
416
+ fig, ax = plt.subplots(figsize=(8, 5))
417
+ ax.hist(max_proba[correct], bins=20, alpha=0.7, color='#22C55E', label='Correct', edgecolor='white')
418
+ ax.hist(max_proba[~correct], bins=20, alpha=0.7, color='#EF4444', label='Incorrect', edgecolor='white')
419
+ ax.set_xlabel('Prediction Confidence')
420
+ ax.set_ylabel('Count')
421
+ ax.set_title('Confidence Distribution: Correct vs Incorrect')
422
+ ax.legend()
423
+ plt.tight_layout()
424
+ path = os.path.join(output_dir, 'confidence_histogram.png')
425
+ plt.savefig(path, dpi=150, bbox_inches='tight')
426
+ plt.close()
427
+ charts.append(path)
428
+ print(f" ✅ Confidence Histogram → {path}")
429
+ except Exception as e:
430
+ print(f" ⚠️ Confidence Histogram failed: {e}")
431
+
432
+ # --- 9. METRICS HEATMAP (per-class precision/recall/f1) ---
433
+ if n_classes <= 10:
434
+ try:
435
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
436
+ prec_per_class = precision_score(y_test, y_pred, average=None, zero_division=0)
437
+ rec_per_class = recall_score(y_test, y_pred, average=None, zero_division=0)
438
+ f1_per_class = f1_score(y_test, y_pred, average=None, zero_division=0)
439
+
440
+ metrics_df = pd.DataFrame({
441
+ 'Precision': prec_per_class[:len(labels)],
442
+ 'Recall': rec_per_class[:len(labels)],
443
+ 'F1-Score': f1_per_class[:len(labels)],
444
+ }, index=labels[:len(prec_per_class)])
445
+
446
+ fig, ax = plt.subplots(figsize=(8, max(3, len(labels) * 0.5)))
447
+ sns.heatmap(metrics_df, annot=True, fmt='.3f', cmap='YlGn', ax=ax,
448
+ vmin=0, vmax=1, linewidths=0.5)
449
+ ax.set_title('Per-Class Metrics Heatmap')
450
+ plt.tight_layout()
451
+ path = os.path.join(output_dir, 'metrics_heatmap.png')
452
+ plt.savefig(path, dpi=150, bbox_inches='tight')
453
+ plt.close()
454
+ charts.append(path)
455
+ print(f" ✅ Metrics Heatmap → {path}")
456
+ except Exception as e:
457
+ print(f" ⚠️ Metrics Heatmap failed: {e}")
458
+
459
+ # --- 10. CLASS BALANCE PIE CHART (actual vs predicted) ---
460
+ try:
461
+ fig, axes = plt.subplots(1, 2, figsize=(12, 5))
462
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
463
+ actual_counts = pd.Series(y_test).value_counts().sort_index()
464
+ pred_counts = pd.Series(y_pred).value_counts().sort_index()
465
+ colors_pie = sns.color_palette('Set2', len(labels))
466
+ axes[0].pie(actual_counts.values, labels=[str(l)[:15] for l in labels],
467
+ autopct='%1.1f%%', colors=colors_pie,
468
+ wedgeprops={'edgecolor': 'white'})
469
+ axes[0].set_title('Actual Class Balance')
470
+ axes[1].pie(pred_counts.reindex(actual_counts.index, fill_value=0).values,
471
+ labels=[str(l)[:15] for l in labels],
472
+ autopct='%1.1f%%', colors=colors_pie,
473
+ wedgeprops={'edgecolor': 'white'})
474
+ axes[1].set_title('Predicted Class Balance')
475
+ plt.suptitle('Class Balance: Actual vs Predicted', fontsize=14)
476
+ plt.tight_layout()
477
+ path = os.path.join(output_dir, 'class_balance_pie.png')
478
+ plt.savefig(path, dpi=150, bbox_inches='tight')
479
+ plt.close()
480
+ charts.append(path)
481
+ print(f" ✅ Class Balance Pie → {path}")
482
+ except Exception as e:
483
+ print(f" ⚠️ Class Balance Pie failed: {e}")
484
+
485
+ # --- 11. CONFIDENCE BOX PLOT BY CLASS ---
486
+ if y_proba is not None:
487
+ try:
488
+ fig, ax = plt.subplots(figsize=(10, 6))
489
+ max_proba = np.max(y_proba, axis=1) if y_proba.ndim == 2 else y_proba
490
+ classes = sorted(np.unique(y_test).tolist())
491
+ data_boxes = [max_proba[y_test == c] for c in classes]
492
+ bp = ax.boxplot(data_boxes, patch_artist=True)
493
+ box_colors = sns.color_palette('Set2', len(classes))
494
+ for i, patch in enumerate(bp['boxes']):
495
+ patch.set_facecolor(box_colors[i % len(box_colors)])
496
+ patch.set_alpha(0.7)
497
+ labels = [str(class_names[i])[:15] if class_names and i < len(class_names) else str(c) for i, c in enumerate(classes)]
498
+ ax.set_xticklabels(labels, rotation=45, ha='right')
499
+ ax.set_xlabel('Class')
500
+ ax.set_ylabel('Prediction Confidence')
501
+ ax.set_title('Confidence Distribution by Class')
502
+ plt.tight_layout()
503
+ path = os.path.join(output_dir, 'confidence_boxplot.png')
504
+ plt.savefig(path, dpi=150, bbox_inches='tight')
505
+ plt.close()
506
+ charts.append(path)
507
+ print(f" ✅ Confidence Boxplot → {path}")
508
+ except Exception as e:
509
+ print(f" ⚠️ Confidence Boxplot failed: {e}")
510
+
511
+ # --- 12. SCORE DISTRIBUTION (KDE per class) ---
512
+ if y_proba is not None and y_proba.ndim == 2:
513
+ try:
514
+ fig, ax = plt.subplots(figsize=(10, 6))
515
+ for i in range(min(y_proba.shape[1], 5)):
516
+ label = str(class_names[i])[:15] if class_names and i < len(class_names) else f'Class {i}'
517
+ try:
518
+ sns.kdeplot(y_proba[:, i], ax=ax, label=label, lw=2)
519
+ except:
520
+ ax.hist(y_proba[:, i], bins=30, alpha=0.5, label=label, density=True)
521
+ ax.set_xlabel('Prediction Score')
522
+ ax.set_ylabel('Density')
523
+ ax.set_title('Score Distribution by Class')
524
+ ax.legend()
525
+ plt.tight_layout()
526
+ path = os.path.join(output_dir, 'score_distribution.png')
527
+ plt.savefig(path, dpi=150, bbox_inches='tight')
528
+ plt.close()
529
+ charts.append(path)
530
+ print(f" ✅ Score Distribution → {path}")
531
+ except Exception as e:
532
+ print(f" ⚠️ Score Distribution failed: {e}")
533
+
534
+ # =====================================================================
535
+ # REGRESSION-SPECIFIC CHARTS
536
+ # =====================================================================
537
+ if not is_clf and y_test is not None and y_pred is not None:
538
+
539
+ # --- 3. ACTUAL VS PREDICTED ---
540
+ try:
541
+ r2 = r2_score(y_test, y_pred)
542
+ mae = mean_absolute_error(y_test, y_pred)
543
+ rmse = np.sqrt(mean_squared_error(y_test, y_pred))
544
+
545
+ fig, ax = plt.subplots(figsize=(8, 8))
546
+ ax.scatter(y_test, y_pred, alpha=0.5, color='#3B82F6', edgecolors='white', s=50)
547
+ lims = [min(y_test.min(), y_pred.min()), max(y_test.max(), y_pred.max())]
548
+ ax.plot(lims, lims, '--', color='#EF4444', linewidth=2, label='Perfect')
549
+ ax.set_title(f'Actual vs Predicted — R²={r2:.4f}, MAE={mae:.4f}, RMSE={rmse:.4f}')
550
+ ax.set_xlabel('Actual')
551
+ ax.set_ylabel('Predicted')
552
+ ax.legend()
553
+ plt.tight_layout()
554
+ path = os.path.join(output_dir, 'actual_vs_predicted.png')
555
+ plt.savefig(path, dpi=150, bbox_inches='tight')
556
+ plt.close()
557
+ charts.append(path)
558
+ print(f" ✅ Actual vs Predicted → {path}")
559
+ except Exception as e:
560
+ print(f" ⚠️ Actual vs Predicted failed: {e}")
561
+
562
+ # --- 4. RESIDUALS ANALYSIS (2x2 grid) ---
563
+ try:
564
+ residuals = y_test - y_pred
565
+ fig, axes = plt.subplots(2, 2, figsize=(12, 10))
566
+ # Residuals vs Predicted
567
+ axes[0, 0].scatter(y_pred, residuals, alpha=0.5, color='#8B5CF6', s=30)
568
+ axes[0, 0].axhline(y=0, color='#EF4444', linestyle='--', linewidth=2)
569
+ axes[0, 0].set_title('Residuals vs Predicted')
570
+ axes[0, 0].set_xlabel('Predicted'); axes[0, 0].set_ylabel('Residuals')
571
+ # Histogram
572
+ axes[0, 1].hist(residuals, bins=30, color='#06B6D4', edgecolor='white', alpha=0.8)
573
+ axes[0, 1].set_title('Residual Distribution')
574
+ axes[0, 1].set_xlabel('Residual'); axes[0, 1].set_ylabel('Frequency')
575
+ # Q-Q plot
576
+ sorted_res = np.sort(residuals)
577
+ n = len(sorted_res)
578
+ theoretical = np.random.normal(0, 1, n)
579
+ theoretical.sort()
580
+ axes[1, 0].scatter(theoretical, sorted_res, alpha=0.5, color='#F59E0B', s=20)
581
+ lims = [min(theoretical.min(), sorted_res.min()), max(theoretical.max(), sorted_res.max())]
582
+ axes[1, 0].plot(lims, lims, 'r--')
583
+ axes[1, 0].set_title('Q-Q Plot')
584
+ axes[1, 0].set_xlabel('Theoretical Quantiles'); axes[1, 0].set_ylabel('Sample Quantiles')
585
+ # Residuals vs Index
586
+ axes[1, 1].plot(range(len(residuals)), residuals, 'o', alpha=0.5, color='#10B981', markersize=4)
587
+ axes[1, 1].axhline(y=0, color='#EF4444', linestyle='--')
588
+ axes[1, 1].set_title('Residuals vs Index')
589
+ axes[1, 1].set_xlabel('Sample Index'); axes[1, 1].set_ylabel('Residual')
590
+ plt.suptitle('Residuals Analysis', fontsize=14)
591
+ plt.tight_layout()
592
+ path = os.path.join(output_dir, 'residuals_analysis.png')
593
+ plt.savefig(path, dpi=150, bbox_inches='tight')
594
+ plt.close()
595
+ charts.append(path)
596
+ print(f" ✅ Residuals Analysis → {path}")
597
+ except Exception as e:
598
+ print(f" ⚠️ Residuals Analysis failed: {e}")
599
+
600
+ # --- 5. ERROR DISTRIBUTION ---
601
+ try:
602
+ abs_errors = np.abs(y_test - y_pred)
603
+ fig, ax = plt.subplots(figsize=(8, 5))
604
+ ax.hist(abs_errors, bins=30, color='#F97316', edgecolor='white', alpha=0.8, density=True)
605
+ p50 = np.percentile(abs_errors, 50)
606
+ p90 = np.percentile(abs_errors, 90)
607
+ p95 = np.percentile(abs_errors, 95)
608
+ ax.axvline(p50, color='green', linestyle='--', label=f'Median: {p50:.3f}')
609
+ ax.axvline(p90, color='orange', linestyle='--', label=f'P90: {p90:.3f}')
610
+ ax.axvline(p95, color='red', linestyle='--', label=f'P95: {p95:.3f}')
611
+ ax.set_title('Absolute Error Distribution')
612
+ ax.set_xlabel('Absolute Error')
613
+ ax.set_ylabel('Density')
614
+ ax.legend()
615
+ plt.tight_layout()
616
+ path = os.path.join(output_dir, 'error_distribution.png')
617
+ plt.savefig(path, dpi=150, bbox_inches='tight')
618
+ plt.close()
619
+ charts.append(path)
620
+ print(f" ✅ Error Distribution → {path}")
621
+ except Exception as e:
622
+ print(f" ⚠️ Error Distribution failed: {e}")
623
+
624
+ # --- 6. PREDICTION OVERVIEW (sorted with ±2sigma) ---
625
+ try:
626
+ idx_sorted = np.argsort(y_test)
627
+ fig, ax = plt.subplots(figsize=(10, 5))
628
+ ax.plot(range(len(y_test)), y_test[idx_sorted], 'o', color='#3B82F6', markersize=3, label='Actual', alpha=0.7)
629
+ ax.plot(range(len(y_pred)), y_pred[idx_sorted], 'x', color='#EF4444', markersize=3, label='Predicted', alpha=0.7)
630
+ residuals_sorted = y_test[idx_sorted] - y_pred[idx_sorted]
631
+ sigma = np.std(residuals_sorted)
632
+ ax.fill_between(range(len(y_test)), y_pred[idx_sorted] - 2*sigma, y_pred[idx_sorted] + 2*sigma,
633
+ alpha=0.15, color='red', label=f'±2σ ({sigma:.3f})')
634
+ ax.set_title('Prediction Overview (Sorted)')
635
+ ax.set_xlabel('Sorted Sample Index')
636
+ ax.set_ylabel('Value')
637
+ ax.legend()
638
+ plt.tight_layout()
639
+ path = os.path.join(output_dir, 'prediction_overview.png')
640
+ plt.savefig(path, dpi=150, bbox_inches='tight')
641
+ plt.close()
642
+ charts.append(path)
643
+ print(f" ✅ Prediction Overview → {path}")
644
+ except Exception as e:
645
+ print(f" ⚠️ Prediction Overview failed: {e}")
646
+
647
+ # --- 7. ERROR BOX PLOT BY QUANTILES ---
648
+ try:
649
+ abs_errors = np.abs(y_test - y_pred)
650
+ quantiles = np.percentile(y_test, [0, 25, 50, 75, 100])
651
+ groups = []
652
+ q_labels = []
653
+ for qi in range(len(quantiles) - 1):
654
+ mask = (y_test >= quantiles[qi]) & (y_test < quantiles[qi+1])
655
+ if mask.sum() > 0:
656
+ groups.append(abs_errors[mask])
657
+ q_labels.append(f'Q{qi+1}')
658
+ if groups:
659
+ fig, ax = plt.subplots(figsize=(10, 6))
660
+ bp = ax.boxplot(groups, patch_artist=True)
661
+ box_colors = ['#3B82F6', '#22C55E', '#F59E0B', '#EF4444']
662
+ for i, patch in enumerate(bp['boxes']):
663
+ patch.set_facecolor(box_colors[i % len(box_colors)])
664
+ patch.set_alpha(0.7)
665
+ ax.set_xticklabels(q_labels)
666
+ ax.set_xlabel('Target Quantile')
667
+ ax.set_ylabel('Absolute Error')
668
+ ax.set_title('Error Distribution by Target Range')
669
+ plt.tight_layout()
670
+ path = os.path.join(output_dir, 'error_boxplot.png')
671
+ plt.savefig(path, dpi=150, bbox_inches='tight')
672
+ plt.close()
673
+ charts.append(path)
674
+ print(f" ✅ Error Boxplot → {path}")
675
+ except Exception as e:
676
+ print(f" ⚠️ Error Boxplot failed: {e}")
677
+
678
+ # --- 8. SCALE-LOCATION PLOT ---
679
+ try:
680
+ residuals_sl = y_test - y_pred
681
+ sqrt_abs_resid = np.sqrt(np.abs(residuals_sl))
682
+ fig, ax = plt.subplots(figsize=(8, 6))
683
+ ax.scatter(y_pred, sqrt_abs_resid, alpha=0.5, color='#8B5CF6', s=30)
684
+ z = np.polyfit(y_pred, sqrt_abs_resid, 1)
685
+ p = np.poly1d(z)
686
+ ax.plot(np.sort(y_pred), p(np.sort(y_pred)), 'r--', lw=2)
687
+ ax.set_xlabel('Fitted Values')
688
+ ax.set_ylabel('√|Residuals|')
689
+ ax.set_title('Scale-Location Plot')
690
+ plt.tight_layout()
691
+ path = os.path.join(output_dir, 'scale_location.png')
692
+ plt.savefig(path, dpi=150, bbox_inches='tight')
693
+ plt.close()
694
+ charts.append(path)
695
+ print(f" ✅ Scale-Location → {path}")
696
+ except Exception as e:
697
+ print(f" ⚠️ Scale-Location failed: {e}")
698
+
699
+ # =====================================================================
700
+ # DATA EXPLORATION CHARTS (both classification & regression)
701
+ # =====================================================================
702
+ if df is not None:
703
+ y = df[target_col] if target_col in df.columns else None
704
+
705
+ # --- 10. CORRELATION HEATMAP ---
706
+ try:
707
+ numeric_df = df.select_dtypes(include=[np.number])
708
+ if len(numeric_df.columns) >= 2:
709
+ corr = numeric_df.corr()
710
+ fig, ax = plt.subplots(figsize=(min(14, len(corr) + 2), min(12, len(corr) + 1)))
711
+ mask = np.triu(np.ones_like(corr, dtype=bool))
712
+ sns.heatmap(corr, mask=mask, annot=len(corr) <= 15, fmt='.2f',
713
+ cmap='RdBu_r', center=0, ax=ax, square=True, linewidths=0.5)
714
+ ax.set_title('Feature Correlation Heatmap')
715
+ plt.tight_layout()
716
+ path = os.path.join(output_dir, 'correlation_heatmap.png')
717
+ plt.savefig(path, dpi=150, bbox_inches='tight')
718
+ plt.close()
719
+ charts.append(path)
720
+ print(f" ✅ Correlation Heatmap → {path}")
721
+ except Exception as e:
722
+ print(f" ⚠️ Correlation Heatmap failed: {e}")
723
+
724
+ # --- 11. FEATURE DISTRIBUTIONS ---
725
+ try:
726
+ plot_cols = NUMERIC_FEATURES if NUMERIC_FEATURES else df.select_dtypes(include=[np.number]).columns[:9].tolist()
727
+ cols_to_plot = [c for c in plot_cols if c in df.columns][:9]
728
+ if cols_to_plot:
729
+ ncols = min(3, len(cols_to_plot))
730
+ nrows = (len(cols_to_plot) + ncols - 1) // ncols
731
+ fig, axes = plt.subplots(nrows, ncols, figsize=(5 * ncols, 4 * nrows))
732
+ if nrows * ncols == 1:
733
+ axes = np.array([axes])
734
+ axes = axes.flatten()
735
+ for i, col in enumerate(cols_to_plot):
736
+ axes[i].hist(df[col].dropna(), bins=30, color='#4F46E5', edgecolor='white', alpha=0.8)
737
+ axes[i].set_title(col, fontsize=10)
738
+ axes[i].tick_params(labelsize=8)
739
+ for i in range(len(cols_to_plot), len(axes)):
740
+ axes[i].set_visible(False)
741
+ plt.suptitle('Feature Distributions', fontsize=14)
742
+ plt.tight_layout()
743
+ path = os.path.join(output_dir, 'distribution_grid.png')
744
+ plt.savefig(path, dpi=150, bbox_inches='tight')
745
+ plt.close()
746
+ charts.append(path)
747
+ print(f" ✅ Distribution Grid → {path}")
748
+ except Exception as e:
749
+ print(f" ⚠️ Distribution Grid failed: {e}")
750
+
751
+ # --- 12. BOXPLOT GRID ---
752
+ try:
753
+ plot_cols = NUMERIC_FEATURES if NUMERIC_FEATURES else df.select_dtypes(include=[np.number]).columns[:9].tolist()
754
+ cols_to_plot = [c for c in plot_cols if c in df.columns][:9]
755
+ if cols_to_plot:
756
+ ncols = min(3, len(cols_to_plot))
757
+ nrows = (len(cols_to_plot) + ncols - 1) // ncols
758
+ fig, axes = plt.subplots(nrows, ncols, figsize=(5 * ncols, 4 * nrows))
759
+ if nrows * ncols == 1:
760
+ axes = np.array([axes])
761
+ axes = axes.flatten()
762
+ colors = sns.color_palette('Set2', len(cols_to_plot))
763
+ for i, col in enumerate(cols_to_plot):
764
+ axes[i].boxplot(df[col].dropna(), patch_artist=True,
765
+ boxprops=dict(facecolor=colors[i], alpha=0.7))
766
+ axes[i].set_title(col, fontsize=10)
767
+ axes[i].tick_params(labelsize=8)
768
+ for i in range(len(cols_to_plot), len(axes)):
769
+ axes[i].set_visible(False)
770
+ plt.suptitle('Feature Boxplots (Outlier Detection)', fontsize=14)
771
+ plt.tight_layout()
772
+ path = os.path.join(output_dir, 'boxplot_grid.png')
773
+ plt.savefig(path, dpi=150, bbox_inches='tight')
774
+ plt.close()
775
+ charts.append(path)
776
+ print(f" ✅ Boxplot Grid → {path}")
777
+ except Exception as e:
778
+ print(f" ⚠️ Boxplot Grid failed: {e}")
779
+
780
+ # --- 13. TARGET DISTRIBUTION (from raw data) ---
781
+ if y is not None:
782
+ try:
783
+ fig, ax = plt.subplots(figsize=(8, 5))
784
+ if is_clf:
785
+ y.value_counts().plot(kind='bar', ax=ax, color=sns.color_palette('Set2'))
786
+ ax.set_ylabel('Count')
787
+ else:
788
+ ax.hist(y.dropna(), bins=30, color='#06B6D4', edgecolor='white', alpha=0.8)
789
+ ax.set_ylabel('Frequency')
790
+ ax.set_title(f'Target Distribution: price_range')
791
+ ax.set_xlabel('price_range')
792
+ plt.xticks(rotation=45, ha='right')
793
+ plt.tight_layout()
794
+ path = os.path.join(output_dir, 'target_distribution.png')
795
+ plt.savefig(path, dpi=150, bbox_inches='tight')
796
+ plt.close()
797
+ charts.append(path)
798
+ print(f" ✅ Target Distribution → {path}")
799
+ except Exception as e:
800
+ print(f" ⚠️ Target Distribution failed: {e}")
801
+
802
+ # =====================================================================
803
+ # UNSUPERVISED LEARNING — Auto Clustering (KMeans)
804
+ # =====================================================================
805
+ try:
806
+ from sklearn.cluster import KMeans
807
+ from sklearn.preprocessing import StandardScaler as ClusterScaler
808
+ from sklearn.decomposition import PCA
809
+ from sklearn.metrics import silhouette_score as sil_score
810
+
811
+ numeric_df = df.select_dtypes(include=[np.number]).dropna()
812
+ if len(numeric_df) >= 10 and len(numeric_df.columns) >= 2:
813
+ # Scale for clustering
814
+ c_scaler = ClusterScaler()
815
+ X_clust = c_scaler.fit_transform(numeric_df)
816
+
817
+ # Find optimal k using silhouette score
818
+ best_k = 3
819
+ best_sil = -1
820
+ for k in range(2, min(8, len(numeric_df))):
821
+ try:
822
+ km = KMeans(n_clusters=k, random_state=42, n_init=10)
823
+ labels_k = km.fit_predict(X_clust)
824
+ sil_k = sil_score(X_clust, labels_k)
825
+ if sil_k > best_sil:
826
+ best_sil = sil_k
827
+ best_k = k
828
+ except:
829
+ pass
830
+
831
+ # Run clustering with best k
832
+ km_final = KMeans(n_clusters=best_k, random_state=42, n_init=10)
833
+ labels = km_final.fit_predict(X_clust)
834
+
835
+ # PCA for 2D visualization
836
+ pca = PCA(n_components=2)
837
+ X_2d = pca.fit_transform(X_clust)
838
+
839
+ # --- Cluster Scatter Plot ---
840
+ fig, ax = plt.subplots(figsize=(10, 8))
841
+ colors_clust = ['#2563eb', '#16a34a', '#dc2626', '#ea580c', '#9333ea',
842
+ '#0891b2', '#db2777', '#d97706', '#0d9488', '#4f46e5']
843
+ for i in range(best_k):
844
+ mask = labels == i
845
+ ax.scatter(X_2d[mask, 0], X_2d[mask, 1],
846
+ c=colors_clust[i % len(colors_clust)],
847
+ label=f'Cluster {i}', alpha=0.7, s=50,
848
+ edgecolors='white', linewidth=0.5)
849
+ ax.set_xlabel('PCA Component 1', fontweight='bold')
850
+ ax.set_ylabel('PCA Component 2', fontweight='bold')
851
+ ax.set_title(f'KMeans Clustering (k={best_k}, Silhouette={best_sil:.3f})',
852
+ fontweight='bold', pad=15, fontsize=14)
853
+ ax.legend(loc='best')
854
+ ax.grid(alpha=0.3)
855
+ plt.tight_layout()
856
+ path = os.path.join(output_dir, 'cluster_scatter.png')
857
+ plt.savefig(path, dpi=150, bbox_inches='tight')
858
+ plt.close()
859
+ charts.append(path)
860
+ print(f" ✅ Cluster Scatter → {path}")
861
+
862
+ # --- Cluster Distribution Bar Chart ---
863
+ fig, ax = plt.subplots(figsize=(8, 5))
864
+ unique, counts = np.unique(labels, return_counts=True)
865
+ ax.bar([f'Cluster {u}' for u in unique], counts,
866
+ color=[colors_clust[u % len(colors_clust)] for u in unique],
867
+ edgecolor='white')
868
+ ax.set_xlabel('Cluster')
869
+ ax.set_ylabel('Count')
870
+ ax.set_title('Cluster Size Distribution')
871
+ for i, (u, c) in enumerate(zip(unique, counts)):
872
+ ax.text(i, c + 0.5, str(c), ha='center', fontsize=10)
873
+ plt.tight_layout()
874
+ path = os.path.join(output_dir, 'cluster_distribution.png')
875
+ plt.savefig(path, dpi=150, bbox_inches='tight')
876
+ plt.close()
877
+ charts.append(path)
878
+ print(f" ✅ Cluster Distribution → {path}")
879
+ except Exception as e:
880
+ print(f" ⚠️ Clustering charts failed: {e}")
881
+
882
+ print(f"\n📊 Generated {len(charts)} charts in '{output_dir}/'")
883
+ return charts
884
+
885
+
886
+ def main():
887
+ parser = argparse.ArgumentParser(description="Generate charts from your trained model")
888
+ parser.add_argument('--model', type=str, default='model.pkl', help='Path to traditional ML model')
889
+ parser.add_argument('--dl-model', type=str, default='deep_learning_model.pkl', help='Path to DL model')
890
+ parser.add_argument('--mode', type=str, default='auto', choices=['auto', 'traditional', 'dl'],
891
+ help='Which model to visualize (auto uses BEST_MODE)')
892
+ parser.add_argument('--data', type=str, default='data/cleaned_data.csv')
893
+ parser.add_argument('--output', type=str, default='charts')
894
+ args = parser.parse_args()
895
+
896
+ print("\n" + "=" * 60)
897
+ print("📊 DataVision Chart Generator")
898
+ print("=" * 60)
899
+
900
+ # Decide which model to load based on BEST_MODE
901
+ use_dl = False
902
+ state = None
903
+ df = None
904
+
905
+ if args.mode == 'dl' or (args.mode == 'auto' and BEST_MODE == 'deep_learning'):
906
+ state = load_dl_model(args.dl_model)
907
+ if state:
908
+ use_dl = True
909
+ else:
910
+ print("⚠️ DL model not found, falling back to traditional ML model...")
911
+
912
+ if not use_dl and args.mode != 'dl':
913
+ state, df = load_model_and_data(args.model, args.data)
914
+ if state is None and os.path.exists(args.dl_model):
915
+ state = load_dl_model(args.dl_model)
916
+ if state:
917
+ use_dl = True
918
+
919
+ if state is None:
920
+ print("❌ No model found. Provide model.pkl or deep_learning_model.pkl")
921
+ return
922
+
923
+ # Load data if not yet loaded
924
+ if df is None:
925
+ data_path = args.data
926
+ df = pd.read_csv(data_path) if os.path.exists(data_path) else None
927
+
928
+ mode_display = 'deep_learning' if use_dl else state.get('mode', 'traditional')
929
+ model_name = state.get('model_name', state.get('algorithm', 'Unknown'))
930
+
931
+ print(f"\n🏆 Model: {model_name}")
932
+ print(f"📊 Mode: {mode_display}")
933
+ print(f"🎯 Task: {state.get('task_type', TASK_TYPE)}")
934
+ if use_dl:
935
+ print(f"🧠 Architecture: {state.get('algorithm', 'MLP')}")
936
+ if df is not None:
937
+ print(f"📋 Data: {len(df)} rows, {len(df.columns)} columns")
938
+
939
+ generate_charts(state, df, args.output)
940
+ print(f"\n✅ Done! Open '{args.output}/' to view charts.")
941
+
942
+
943
+ if __name__ == '__main__':
944
+ main()
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/Dockerfile ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.10-slim
2
+
3
+ WORKDIR /app
4
+
5
+ COPY requirements.txt .
6
+ RUN pip install --no-cache-dir -r requirements.txt
7
+
8
+ COPY . .
9
+
10
+ EXPOSE 5000
11
+
12
+ CMD ["python", "api_server.py", "--port", "5000"]
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/README.md ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚀 DataVision ML Project — Complete Guide
2
+
3
+ Auto-generated by **DataVision AI Business Analyst**
4
+ *This ZIP contains YOUR actual trained model, data, scripts, and everything needed to use it.*
5
+
6
+ ---
7
+
8
+ ## 📦 What's Inside This ZIP?
9
+
10
+ ```
11
+ ├── model.pkl # Your trained model
12
+ ├── data/
13
+ │ └── cleaned_data.csv # Your cleaned dataset
14
+ ├── predict.py # Make predictions (CLI / import)
15
+ ├── visualize.py # Generate performance charts
16
+ ├── evaluate.py # Comprehensive model evaluation
17
+ ├── train_traditional.py # Traditional ML training
18
+ ├── charts/ # Pre-generated training charts (PNG)
19
+ ├── api_server.py # Flask REST API for deployment
20
+ ├── utils/
21
+ │ └── preprocessing.py # Data preprocessing utilities
22
+ ├── config.json # Full training configuration
23
+ ├── requirements.txt # Python dependencies
24
+ ├── Dockerfile # Docker deployment
25
+ └── README.md # This file
26
+ ```
27
+
28
+ ### What Each File Does:
29
+
30
+ | File | What It Does | When To Use |
31
+ |------|-------------|-------------|
32
+ | `model.pkl` | Your **already trained** ML model (ready to use) | Loaded automatically by other scripts |
33
+ | `data/cleaned_data.csv` | Your cleaned dataset (used for charts & evaluation) | Used by visualize.py and evaluate.py |
34
+ | `predict.py` | Make predictions on new data | When you want to predict something |
35
+ | `visualize.py` | Generate all charts (confusion matrix, ROC, etc.) | When you want to see model performance charts |
36
+ | `evaluate.py` | Show detailed model metrics and scores | When you want to see accuracy, F1, R2 scores |
37
+ | `api_server.py` | Start a web API server with a form | When you want a web interface or REST API |
38
+ | `requirements.txt` | List of Python packages needed | Install once before running anything |
39
+ | `config.json` | Full training configuration and metadata | Reference only |
40
+ | `README.md` | This guide | You are reading it now |
41
+
42
+ ---
43
+
44
+ ## Step-by-Step: Getting Started (For Beginners)
45
+
46
+ ### STEP 1: Install Python (if not installed)
47
+
48
+ Download and install Python 3.8 or newer from https://www.python.org/downloads/
49
+
50
+ > **Important**: During installation, check the box **"Add Python to PATH"**
51
+
52
+ ### STEP 2: Open Terminal / Command Prompt
53
+
54
+ - **Windows**: Press `Win + R`, type `cmd`, press Enter
55
+ - **Mac/Linux**: Open Terminal app
56
+
57
+ Navigate to the project folder:
58
+ ```bash
59
+ cd path/to/this/folder
60
+ ```
61
+
62
+ ### STEP 3: Install Dependencies (Run Once)
63
+
64
+ ```bash
65
+ pip install -r requirements.txt
66
+ ```
67
+
68
+ This installs all the Python packages needed. You only need to do this **once**.
69
+
70
+ > If you get an error, try: `pip3 install -r requirements.txt`
71
+
72
+ ---
73
+
74
+ ## STEP 4: See Your Model Results (evaluate.py)
75
+
76
+ This shows your **exact same scores** from DataVision.
77
+ Automatically evaluates the correct model (Traditional ML or Deep Learning) based on your best training mode.
78
+
79
+ ```bash
80
+ python evaluate.py # Auto-detect best model
81
+ python evaluate.py --mode dl # Force Deep Learning model
82
+ python evaluate.py --mode traditional # Force traditional ML model
83
+ ```
84
+
85
+ **What you will see:**
86
+ - Your model accuracy or R2 score
87
+ - Classification report (precision, recall, F1 per class)
88
+ - Confusion matrix
89
+ - All stored metrics from training
90
+
91
+ **Additional options:**
92
+ ```bash
93
+ python evaluate.py --cv 5 # Run 5-fold cross-validation
94
+ python evaluate.py --reeval # Re-evaluate model on dataset
95
+ ```
96
+
97
+ **Expected output example:**
98
+ ```
99
+ Model: RandomForest
100
+ Task: classification
101
+ Accuracy: 0.9450
102
+ F1-Score: 0.9380
103
+ ```
104
+
105
+ ---
106
+
107
+ ## STEP 5: Make Predictions (predict.py)
108
+
109
+ ### Option A: Interactive Mode (Easiest)
110
+ ```bash
111
+ python predict.py
112
+ ```
113
+ It will ask you to enter a value for each feature one by one, then show the prediction.
114
+
115
+ ### Option B: JSON Input (Single Prediction)
116
+ ```bash
117
+ python predict.py --json "{\"feature1\": 25, \"feature2\": \"value\"}"
118
+ ```
119
+ Replace `feature1`, `feature2` with your actual column names.
120
+
121
+ ### Option C: CSV Batch Predictions (Many Rows)
122
+ ```bash
123
+ python predict.py --csv your_data.csv
124
+ ```
125
+ This reads your CSV, predicts for every row, saves results to `your_data_predictions.csv`.
126
+
127
+ ```bash
128
+ python predict.py --csv input.csv --output results.csv
129
+ ```
130
+
131
+ **Expected output example:**
132
+ ```
133
+ Prediction: Normal
134
+ Confidence: 94.5%
135
+ Probabilities:
136
+ Normal: 94.5%
137
+ Abnormal: 5.5%
138
+ ```
139
+
140
+ ---
141
+
142
+ ## STEP 6: Generate Charts (visualize.py)
143
+
144
+ Creates **all the same charts** you saw in DataVision AutoML.
145
+ Automatically loads the correct model (Traditional ML or Deep Learning) based on your best training mode.
146
+
147
+ ```bash
148
+ python visualize.py # Auto-detect best model
149
+ python visualize.py --mode dl # Force Deep Learning model
150
+ python visualize.py --mode traditional # Force traditional ML model
151
+ ```
152
+
153
+ Charts are saved to a `charts/` folder. Open them with any image viewer.
154
+
155
+ **Classification charts generated:**
156
+
157
+ | Chart File | What It Shows |
158
+ |------------|---------------|
159
+ | `model_comparison.png` | All models ranked by score |
160
+ | `feature_importance.png` | Which features matter most |
161
+ | `confusion_matrix.png` | Correct vs wrong predictions per class |
162
+ | `class_distribution.png` | Actual vs predicted class counts |
163
+ | `roc_curve.png` | ROC curve with AUC score |
164
+ | `precision_recall.png` | Precision-Recall curve |
165
+ | `calibration.png` | Probability calibration curve |
166
+ | `confidence_histogram.png` | Confidence for correct vs incorrect |
167
+ | `metrics_heatmap.png` | Per-class precision, recall, F1 |
168
+ | `class_balance_pie.png` | Class balance pie charts |
169
+ | `confidence_boxplot.png` | Confidence box plot per class |
170
+ | `score_distribution.png` | Score distribution per class |
171
+
172
+ **Regression charts generated:**
173
+
174
+ | Chart File | What It Shows |
175
+ |------------|---------------|
176
+ | `actual_vs_predicted.png` | Scatter: actual vs predicted values |
177
+ | `residuals_analysis.png` | 4-panel residual analysis |
178
+ | `error_distribution.png` | Error distribution with percentiles |
179
+ | `prediction_overview.png` | Sorted predictions with error band |
180
+ | `error_boxplot.png` | Error by target range (Q1-Q4) |
181
+ | `scale_location.png` | Scale-Location diagnostic plot |
182
+
183
+ **Common charts (both):**
184
+
185
+ | Chart File | What It Shows |
186
+ |------------|---------------|
187
+ | `correlation_heatmap.png` | Feature correlations |
188
+ | `distribution_grid.png` | Distribution of each feature |
189
+ | `boxplot_grid.png` | Outlier detection per feature |
190
+ | `target_distribution.png` | Target column distribution |
191
+
192
+ **Deep Learning charts (when DL model is loaded):**
193
+
194
+ | Chart File | What It Shows |
195
+ |------------|---------------|
196
+ | `training_history.png` | Training loss curve and validation scores |
197
+ | `dl_model_summary.png` | Model architecture, hyperparameters, metrics |
198
+ | `feature_importance.png` | MLP weight-based feature importance |
199
+
200
+ **Unsupervised Learning charts (auto-generated):**
201
+
202
+ | Chart File | What It Shows |
203
+ |------------|---------------|
204
+ | `cluster_scatter.png` | KMeans clustering with PCA 2D projection |
205
+ | `cluster_distribution.png` | Cluster size distribution |
206
+
207
+ > **Note:** Your ZIP already contains pre-generated charts in the `charts/` folder
208
+ > from your actual training. These are the EXACT same charts you saw in DataVision.
209
+ > Running `visualize.py` will regenerate them from your model data.
210
+
211
+ **Custom options:**
212
+ ```bash
213
+ python visualize.py --data other_data.csv # Use different data
214
+ python visualize.py --output my_charts/ # Save to different folder
215
+ ```
216
+
217
+ ---
218
+
219
+ ## STEP 7: Deploy as Web API (api_server.py)
220
+
221
+ Start a local web server with a prediction form:
222
+
223
+ ```bash
224
+ python api_server.py
225
+ ```
226
+
227
+ Then open your browser and go to: **http://localhost:5000/predict**
228
+
229
+ You will see a **web form** with your actual column names where you can:
230
+ 1. Enter values for each feature
231
+ 2. Click "Predict"
232
+ 3. See the prediction result instantly
233
+
234
+ **API Endpoints:**
235
+
236
+ | URL | Method | What It Does |
237
+ |-----|--------|-------------|
238
+ | `/predict` | GET | Web form with your feature names |
239
+ | `/predict` | POST | JSON API for developers |
240
+ | `/health` | GET | Health check (is server running?) |
241
+ | `/model-info` | GET | Model metadata and metrics |
242
+
243
+ **Change port:**
244
+ ```bash
245
+ python api_server.py --port 8080
246
+ ```
247
+
248
+ ---
249
+
250
+ ## STEP 8: Retrain the Model (Optional)
251
+
252
+ > Your model is already trained in `model.pkl`. Only retrain if you want to
253
+ > experiment with new data or verify the training.
254
+
255
+ #### Traditional ML
256
+ ```bash
257
+ python train_traditional.py # Reproduces your training
258
+ python train_traditional.py --data your.csv # Custom data
259
+ ```
260
+
261
+ ---
262
+
263
+ ## STEP 9: Docker Deployment (Advanced)
264
+
265
+ ```bash
266
+ docker build -t my-ml-model .
267
+ docker run -p 5000:5000 my-ml-model
268
+ ```
269
+
270
+ Then access: `http://localhost:5000/predict`
271
+
272
+ ---
273
+
274
+ ## Your Model Summary
275
+
276
+ | Property | Value |
277
+ |----------|-------|
278
+ | **Target Column** | `price_range` |
279
+ | **Task Type** | classification |
280
+ | **Best Model** | RandomForest |
281
+ | **Training Mode** | traditional |
282
+ | **Features** | 20 columns |
283
+ | **Dataset Rows** | 2000 |
284
+ | **Modes Trained** | traditional |
285
+
286
+ ### Training Metrics
287
+ - **ACCURACY**: 0.8925 (89.2%)
288
+ - **F1**: 0.8929 (89.3%)
289
+ - **PRECISION**: 0.8937 (89.4%)
290
+ - **RECALL**: 0.8925 (89.2%)
291
+ - **ROC_AUC**: 0.9796 (98.0%)
292
+
293
+ ### Model Leaderboard
294
+
295
+ | Rank | Model | Score |
296
+ |------|-------|-------|
297
+ | 1 | ? | 0.8925 |
298
+
299
+ ### Feature Details
300
+
301
+ | Feature | Type | Range/Values |
302
+ |---------|------|-------------|
303
+ | `battery_power` | numeric | 510.0 - 1987.0 |
304
+ | `blue` | numeric | 0.0 - 1.0 |
305
+ | `clock_speed` | numeric | 0.5 - 3.0 |
306
+ | `dual_sim` | numeric | 0.0 - 1.0 |
307
+ | `fc` | numeric | 0.0 - 16.0 |
308
+ | `four_g` | numeric | 0.0 - 1.0 |
309
+ | `int_memory` | numeric | 2.0 - 64.0 |
310
+ | `m_dep` | numeric | 0.1 - 1.0 |
311
+ | `mobile_wt` | numeric | 80.0 - 199.0 |
312
+ | `n_cores` | numeric | 1.0 - 8.0 |
313
+
314
+ ---
315
+
316
+ ## Troubleshooting
317
+
318
+ | Problem | Solution |
319
+ |---------|----------|
320
+ | `ModuleNotFoundError` | Run `pip install -r requirements.txt` |
321
+ | `model.pkl not found` | Make sure you are in the project folder |
322
+ | `No module named flask` | Run `pip install flask` |
323
+ | `Port already in use` | Use `python api_server.py --port 8080` |
324
+ | Charts not generating | Make sure `data/cleaned_data.csv` exists |
325
+ | Wrong model loaded | Use `--mode dl` or `--mode traditional` to force specific model |
326
+ | DL model not found | Make sure `deep_learning_model.pkl` exists |
327
+ | Permission denied | On Mac/Linux: try `python3` instead of `python` |
328
+
329
+ ---
330
+
331
+ ## Using as Python Module (For Developers)
332
+
333
+ ```python
334
+ from predict import load_model, predict_single
335
+
336
+ state = load_model('model.pkl')
337
+ result = predict_single(state, {
338
+ # Use your actual column names here
339
+ # Run 'python predict.py' to see what columns are needed
340
+ })
341
+ print(f"Prediction: {result['prediction']}")
342
+ if 'confidence' in result:
343
+ print(f"Confidence: {result['confidence']:.1%}")
344
+ ```
345
+
346
+ ---
347
+ *Generated by DataVision AI Business Analyst*
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/api_server.py ADDED
@@ -0,0 +1,633 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🌐 Flask API Server — Your Trained Model
3
+ ============================================
4
+ Auto-generated by DataVision AI
5
+
6
+ Model : RandomForest
7
+ Target Column : price_range
8
+ Task Type : classification
9
+ Features : 20 columns
10
+
11
+ Usage:
12
+ python api_server.py
13
+ python api_server.py --port 8080
14
+
15
+ Endpoints:
16
+ POST /predict — Make a prediction (JSON body with your feature names)
17
+ GET /health — Health check
18
+ GET /model-info — Model metadata and expected features
19
+ """
20
+
21
+ import os
22
+ import json
23
+ import re
24
+ import pickle
25
+ import argparse
26
+ import numpy as np
27
+ import pandas as pd
28
+ from flask import Flask, request, jsonify
29
+ from sklearn.preprocessing import LabelEncoder, RobustScaler
30
+
31
+ app = Flask(__name__)
32
+
33
+ MODEL_STATE = None
34
+ FEATURE_ENGINEER = None
35
+ FEATURE_COLUMNS = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt', 'n_cores', 'pc', 'px_height', 'px_width', 'ram', 'sc_h', 'sc_w', 'talk_time', 'three_g', 'touch_screen', 'wifi']
36
+ TARGET_COLUMN = 'price_range'
37
+ BEST_MODE = 'traditional' # Which mode won during AutoML
38
+
39
+ # YOUR original column names with types and ranges (what you input)
40
+ RAW_FEATURE_INFO = {
41
+ "battery_power": {"type": "numeric", "min": 510.0, "max": 1987.0, "default": 1238.51},
42
+ "blue": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.49},
43
+ "clock_speed": {"type": "numeric", "min": 0.5, "max": 3.0, "default": 1.52},
44
+ "dual_sim": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
45
+ "fc": {"type": "numeric", "min": 0.0, "max": 16.0, "default": 4.29},
46
+ "four_g": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.52},
47
+ "int_memory": {"type": "numeric", "min": 2.0, "max": 64.0, "default": 32.05},
48
+ "m_dep": {"type": "numeric", "min": 0.1, "max": 1.0, "default": 0.50},
49
+ "mobile_wt": {"type": "numeric", "min": 80.0, "max": 199.0, "default": 140.24},
50
+ "n_cores": {"type": "numeric", "min": 1.0, "max": 8.0, "default": 4.52},
51
+ "pc": {"type": "numeric", "min": 0.0, "max": 20.0, "default": 9.92},
52
+ "px_height": {"type": "numeric", "min": 15.0, "max": 1791.01, "default": 644.46},
53
+ "ram": {"type": "numeric", "min": 296.99, "max": 3958.01, "default": 2124.26},
54
+ "sc_h": {"type": "numeric", "min": 5.0, "max": 19.0, "default": 12.31},
55
+ "sc_w": {"type": "numeric", "min": 0.0, "max": 17.0, "default": 5.76},
56
+ "three_g": {"type": "numeric", "min": 1.0, "max": 1.0, "default": 1.00},
57
+ "touch_screen": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.50},
58
+ "wifi": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
59
+ }
60
+
61
+
62
+ # =============================================================================
63
+ # Standalone Feature Engineer — exact DataVision preprocessing
64
+ # =============================================================================
65
+ class StandaloneFeatureEngineer:
66
+ def __init__(self, state):
67
+ self.transformers = state.get('transformers', {})
68
+ self.encoders = state.get('encoders', {})
69
+ self.feature_names = state.get('feature_names', [])
70
+ self.original_columns = state.get('original_columns', [])
71
+ self.selected_feature_indices = state.get('selected_feature_indices', None)
72
+ self.is_fitted = state.get('is_fitted', True)
73
+ self.stopwords = set("i me my myself we our ours ourselves you your yours yourself yourselves he him his himself she her hers herself it its itself they them their theirs themselves what which who whom this that these those am is are was were be been being have has had having do does did doing a an the and but if or because as until while of at by for with about against between through during before after above below to from up down in out on off over under again further then once here there when where why how all both each few more most other some such no nor not only own same so than too very s t can will just don should now d ll m o re ve y ain aren couldn didn doesn hadn hasn haven isn ma mightn mustn needn shan shouldn wasn weren won wouldn".split())
74
+ self.positive_words = set("good great excellent amazing wonderful fantastic brilliant outstanding superb love best perfect beautiful awesome incredible impressive magnificent exceptional fabulous terrific marvelous stellar remarkable extraordinary delightful gorgeous splendid spectacular phenomenal exquisite lovely charming pleasant nice fine happy joy enjoy satisfied pleased glad cheerful excited thrilled".split())
75
+ self.negative_words = set("bad terrible horrible awful worst poor disappointing disgusting pathetic useless dreadful atrocious appalling abysmal mediocre inferior unpleasant nasty vile wretched miserable lousy horrendous hideous ghastly hideous deplorable lamentable pitiful woeful tragic sad angry frustrated annoyed irritated disappointed upset unhappy distressed sorrowful gloomy depressed".split())
76
+
77
+ def _clean_text(self, text):
78
+ text = str(text).lower()
79
+ text = re.sub(r'<[^>]+>', ' ', text)
80
+ text = re.sub(r'http\S+', ' ', text)
81
+ text = re.sub(r'[^a-z0-9\s]', ' ', text)
82
+ text = ' '.join(w for w in text.split() if w not in self.stopwords and len(w) > 1)
83
+ return text if text.strip() else 'empty'
84
+
85
+ def _text_statistics(self, text):
86
+ text = str(text)
87
+ words = text.split()
88
+ wl = [len(w) for w in words] if words else [0]
89
+ return [len(text), len(words), np.mean(wl), np.std(wl), max(wl) if wl else 0,
90
+ len(set(words)) / max(len(words), 1), text.count('!'), text.count('?'),
91
+ sum(1 for c in text if c.isupper()) / max(len(text), 1),
92
+ len([w for w in words if len(w) > 6]) / max(len(words), 1)]
93
+
94
+ def _sentiment_score(self, text):
95
+ words = set(str(text).lower().split())
96
+ pos = len(words & self.positive_words)
97
+ neg = len(words & self.negative_words)
98
+ total = max(pos + neg, 1)
99
+ return [pos / total, neg / total, (pos - neg) / total, total]
100
+
101
+ def transform(self, df):
102
+ df = df.copy()
103
+ cols = df.columns.tolist()
104
+ seen = {}
105
+ new_cols = []
106
+ for col in cols:
107
+ if col in seen:
108
+ seen[col] += 1
109
+ new_cols.append(f"{col}_{seen[col]}")
110
+ else:
111
+ seen[col] = 0
112
+ new_cols.append(col)
113
+ df.columns = new_cols
114
+ feature_parts = []
115
+
116
+ if 'numeric_cols' in self.transformers:
117
+ numeric_cols = self.transformers['numeric_cols']
118
+ for col in numeric_cols:
119
+ if col not in df.columns:
120
+ df[col] = 0
121
+ try:
122
+ numeric_data = df[numeric_cols].apply(pd.to_numeric, errors='coerce').fillna(0).values
123
+ scaler = self.transformers['numeric_scaler']
124
+ numeric_scaled = scaler.transform(numeric_data)
125
+ numeric_scaled = np.nan_to_num(numeric_scaled, nan=0.0, posinf=0.0, neginf=0.0)
126
+ feature_parts.append(numeric_scaled)
127
+ if 2 <= len(numeric_cols) <= 10:
128
+ interactions = []
129
+ for i in range(min(len(numeric_cols), 4)):
130
+ interactions.append((numeric_scaled[:, i] ** 2).reshape(-1, 1))
131
+ for i in range(min(len(numeric_cols), 4)):
132
+ for j in range(i + 1, min(len(numeric_cols), 4)):
133
+ interactions.append((numeric_scaled[:, i] * numeric_scaled[:, j]).reshape(-1, 1))
134
+ if interactions:
135
+ feature_parts.append(np.hstack(interactions))
136
+ except:
137
+ feature_parts.append(np.zeros((len(df), len(numeric_cols))))
138
+
139
+ for key, cols in self.transformers.items():
140
+ if key.endswith('_onehot'):
141
+ col_name = key.replace('_onehot', '')
142
+ try:
143
+ if col_name in df.columns:
144
+ series = df[col_name].fillna('_MISSING_').astype(str).str.strip()
145
+ dummies = pd.get_dummies(series, prefix=col_name)
146
+ dummies = dummies.reindex(columns=cols, fill_value=0)
147
+ feature_parts.append(dummies.values.astype(float))
148
+ else:
149
+ feature_parts.append(np.zeros((len(df), len(cols))))
150
+ except:
151
+ feature_parts.append(np.zeros((len(df), len(cols))))
152
+
153
+ for col, le in self.encoders.items():
154
+ try:
155
+ if col in df.columns:
156
+ series = df[col].fillna('_MISSING_').astype(str).str.strip()
157
+ encoded = series.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
158
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
159
+ freq_key = f'{col}_freq_map'
160
+ if freq_key in self.transformers:
161
+ freq_map = self.transformers[freq_key]
162
+ freq_encoded = series.map(lambda s: freq_map.get(s, 0.0)).values.astype(float)
163
+ feature_parts.append(freq_encoded.reshape(-1, 1))
164
+ else:
165
+ feature_parts.append(np.zeros((len(df), 1)))
166
+ if f'{col}_freq_map' in self.transformers:
167
+ feature_parts.append(np.zeros((len(df), 1)))
168
+ except:
169
+ feature_parts.append(np.zeros((len(df), 1)))
170
+
171
+ if 'cat_interactions' in self.transformers:
172
+ for key, value in self.transformers['cat_interactions'].items():
173
+ if key.endswith('_freq_map'):
174
+ continue
175
+ try:
176
+ col1, col2, le = value
177
+ if col1 in df.columns and col2 in df.columns:
178
+ combined = df[col1].fillna('_NA_').astype(str) + "_X_" + df[col2].fillna('_NA_').astype(str)
179
+ encoded = combined.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
180
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
181
+ fk = f"{key}_freq_map"
182
+ if fk in self.transformers['cat_interactions']:
183
+ fm = self.transformers['cat_interactions'][fk]
184
+ feature_parts.append(combined.map(lambda s: fm.get(s, 0.0)).values.reshape(-1, 1).astype(float))
185
+ else:
186
+ feature_parts.append(np.zeros((len(df), 1)))
187
+ except:
188
+ pass
189
+
190
+ if 'num_cat_aggs' in self.transformers:
191
+ for key, mean_map in self.transformers['num_cat_aggs'].items():
192
+ try:
193
+ parts = key.rsplit('_by_', 1)
194
+ if len(parts) == 2:
195
+ num_col, cat_col = parts[0], parts[1].rsplit('_mean', 1)[0]
196
+ if num_col in df.columns and cat_col in df.columns:
197
+ cat_s = df[cat_col].fillna('_NA_').astype(str)
198
+ dv = np.mean(list(mean_map.values())) if mean_map else 0
199
+ agg = cat_s.map(lambda s: mean_map.get(s, dv)).values.astype(float)
200
+ feature_parts.append(agg.reshape(-1, 1))
201
+ feature_parts.append((df[num_col].fillna(0).values - agg).reshape(-1, 1))
202
+ else:
203
+ feature_parts.extend([np.zeros((len(df), 1))] * 2)
204
+ except:
205
+ pass
206
+
207
+ for key, tfidf in self.transformers.items():
208
+ if key.endswith('_tfidf'):
209
+ col = key.replace('_tfidf', '')
210
+ try:
211
+ if col in df.columns:
212
+ series = df[col].fillna('').astype(str)
213
+ cleaned = series.apply(self._clean_text)
214
+ tm = tfidf.transform(cleaned)
215
+ svd_key = f'{col}_svd'
216
+ feature_parts.append(self.transformers[svd_key].transform(tm) if svd_key in self.transformers else tm.toarray())
217
+ feature_parts.append(np.array([self._text_statistics(t) for t in series]))
218
+ feature_parts.append(np.array([self._sentiment_score(t) for t in series]))
219
+ except:
220
+ pass
221
+
222
+ if not feature_parts:
223
+ return np.zeros((len(df), len(self.feature_names)))
224
+ X = np.hstack(feature_parts)
225
+ X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
226
+
227
+ if 'variance_selector' in self.transformers:
228
+ sel = self.transformers['variance_selector']
229
+ try:
230
+ if X.shape[1] == sel.n_features_in_:
231
+ X = sel.transform(X)
232
+ elif self.selected_feature_indices is not None:
233
+ vi = [i for i in self.selected_feature_indices if i < X.shape[1]]
234
+ if vi:
235
+ X = X[:, vi]
236
+ except:
237
+ pass
238
+
239
+ if 'pca' in self.transformers:
240
+ try:
241
+ pca = self.transformers['pca']
242
+ if X.shape[1] == pca.n_features_in_:
243
+ X = pca.transform(X)
244
+ except:
245
+ pass
246
+
247
+ if X.shape[1] != len(self.feature_names):
248
+ if X.shape[1] < len(self.feature_names):
249
+ X = np.hstack([X, np.zeros((len(df), len(self.feature_names) - X.shape[1]))])
250
+ else:
251
+ X = X[:, :len(self.feature_names)]
252
+ return X
253
+
254
+ def transform_single(self, data):
255
+ for col in self.original_columns:
256
+ if col not in data:
257
+ data[col] = 0
258
+ df = pd.DataFrame([data])
259
+ X = self.transform(df)
260
+ n = len(self.feature_names)
261
+ if X.shape[1] < n:
262
+ X = np.hstack([X, np.zeros((1, n - X.shape[1]))])
263
+ elif X.shape[1] > n:
264
+ X = X[:, :n]
265
+ return X
266
+
267
+
268
+ def load_model():
269
+ global MODEL_STATE, FEATURE_ENGINEER
270
+
271
+ # If best mode is deep_learning, load DL first as primary
272
+ if BEST_MODE == 'deep_learning':
273
+ load_dl_model()
274
+ if DL_STATE is not None:
275
+ print(f"🧠 Primary mode: Deep Learning (BEST_MODE={BEST_MODE})")
276
+ # Also try loading ML model as fallback
277
+ for path in ['model.pkl', 'nlp_model.pkl']:
278
+ if os.path.exists(path):
279
+ with open(path, 'rb') as f:
280
+ MODEL_STATE = pickle.load(f)
281
+ print(f" Also loaded fallback: {path}")
282
+ break
283
+ return True
284
+
285
+ # Traditional/NLP: load ML model first
286
+ for path in ['model.pkl', 'nlp_model.pkl']:
287
+ if os.path.exists(path):
288
+ with open(path, 'rb') as f:
289
+ MODEL_STATE = pickle.load(f)
290
+ print(f"✅ Loaded: {path} ({MODEL_STATE.get('model_name', 'Unknown')})")
291
+ # Load feature engineer for exact preprocessing
292
+ eng_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'feature_engineer.pkl')
293
+ if os.path.exists(eng_path):
294
+ try:
295
+ with open(eng_path, 'rb') as ef:
296
+ eng_state = pickle.load(ef)
297
+ FEATURE_ENGINEER = StandaloneFeatureEngineer(eng_state)
298
+ print(f" Pipeline: feature_engineer.pkl ({len(FEATURE_ENGINEER.feature_names)} features)")
299
+ except Exception as e:
300
+ print(f" Warning: Could not load feature_engineer.pkl: {e}")
301
+ break
302
+
303
+ # Also load DL model if available
304
+ load_dl_model()
305
+
306
+ if MODEL_STATE is None and DL_STATE is None:
307
+ print("❌ No model file found")
308
+ return False
309
+ return True
310
+
311
+
312
+ # =============================================================================
313
+ # Deep Learning Model Support (sklearn MLPClassifier/MLPRegressor)
314
+ # =============================================================================
315
+ DL_STATE = None
316
+
317
+ def load_dl_model():
318
+ """Load the Deep Learning model if available."""
319
+ global DL_STATE
320
+ dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'deep_learning_model.pkl')
321
+ if os.path.exists(dl_path):
322
+ try:
323
+ with open(dl_path, 'rb') as f:
324
+ DL_STATE = pickle.load(f)
325
+ print(f"✅ Loaded DL model: {DL_STATE.get('algorithm', 'MLP')}")
326
+ except Exception as e:
327
+ print(f"⚠️ Failed to load DL model: {e}")
328
+
329
+
330
+ def preprocess_dl_input(data):
331
+ """Preprocess input for Deep Learning model using feature_metadata."""
332
+ feature_metadata = DL_STATE.get('feature_metadata', [])
333
+ feature_values = []
334
+
335
+ # Process numeric features
336
+ for meta in feature_metadata:
337
+ if meta.get('type') == 'numeric':
338
+ col = meta['name']
339
+ if col in data:
340
+ try:
341
+ feature_values.append(float(data[col]))
342
+ except (ValueError, TypeError):
343
+ feature_values.append(meta.get('mean', 0))
344
+ else:
345
+ feature_values.append(meta.get('mean', 0))
346
+
347
+ # Process categorical features (one-hot)
348
+ for meta in feature_metadata:
349
+ if meta.get('type') == 'categorical':
350
+ col = meta['name']
351
+ value = str(data.get(col, ''))
352
+ options = meta.get('options', [])
353
+ for opt in options:
354
+ feature_values.append(1.0 if value == opt else 0.0)
355
+ feature_values.append(1.0 if not value or value == 'nan' else 0.0)
356
+
357
+ # Fallback
358
+ if not feature_values and DL_STATE.get('feature_columns'):
359
+ for col in DL_STATE['feature_columns']:
360
+ if col in data:
361
+ try:
362
+ feature_values.append(float(data[col]) if not isinstance(data[col], str) else 0)
363
+ except:
364
+ feature_values.append(0)
365
+ else:
366
+ feature_values.append(0)
367
+
368
+ X = np.array([feature_values])
369
+
370
+ scaler = DL_STATE.get('scaler')
371
+ if scaler and hasattr(scaler, 'n_features_in_'):
372
+ expected = scaler.n_features_in_
373
+ actual = X.shape[1]
374
+ if actual < expected:
375
+ X = np.hstack([X, np.zeros((1, expected - actual))])
376
+ elif actual > expected:
377
+ X = X[:, :expected]
378
+
379
+ if scaler:
380
+ X = scaler.transform(X)
381
+
382
+ return X
383
+
384
+
385
+ def preprocess_input(data):
386
+ """Preprocess raw input using the EXACT same pipeline as DataVision AutoML."""
387
+ global FEATURE_ENGINEER
388
+
389
+ # Use feature engineer (exact AutoML pipeline)
390
+ if FEATURE_ENGINEER is not None:
391
+ try:
392
+ return FEATURE_ENGINEER.transform_single(data)
393
+ except Exception as e:
394
+ print(f"Warning: Feature engineer failed ({e}), using fallback")
395
+
396
+ # Fallback: basic label-encode + scale
397
+ feature_cols = MODEL_STATE.get('feature_columns', FEATURE_COLUMNS)
398
+ label_encoders = MODEL_STATE.get('label_encoders', {})
399
+ scaler = MODEL_STATE.get('scaler')
400
+
401
+ row = {}
402
+ for col in feature_cols:
403
+ val = data.get(col, 0)
404
+ if col in label_encoders:
405
+ le = label_encoders[col]
406
+ val_str = str(val)
407
+ if val_str in le.classes_:
408
+ val = le.transform([val_str])[0]
409
+ else:
410
+ val = 0
411
+ else:
412
+ try:
413
+ val = float(val)
414
+ except (ValueError, TypeError):
415
+ val = 0
416
+ row[col] = val
417
+
418
+ X = pd.DataFrame([row], columns=feature_cols)
419
+ if scaler:
420
+ try:
421
+ X = pd.DataFrame(scaler.transform(X), columns=feature_cols)
422
+ except:
423
+ pass
424
+ return X
425
+
426
+
427
+ @app.route('/health', methods=['GET'])
428
+ def health():
429
+ return jsonify({"status": "healthy", "model_loaded": MODEL_STATE is not None or DL_STATE is not None, "dl_model_loaded": DL_STATE is not None})
430
+
431
+
432
+ @app.route('/model-info', methods=['GET'])
433
+ def model_info():
434
+ if MODEL_STATE is None and DL_STATE is None:
435
+ return jsonify({"error": "No model loaded"}), 404
436
+
437
+ info = {"best_mode": BEST_MODE}
438
+ if BEST_MODE == 'deep_learning' and DL_STATE:
439
+ info.update({
440
+ "model_name": DL_STATE.get('algorithm', 'MLP Neural Network'),
441
+ "task_type": DL_STATE.get('task_type', 'classification'),
442
+ "target_column": TARGET_COLUMN,
443
+ "input_features": list(RAW_FEATURE_INFO.keys()),
444
+ "metrics": {k: float(v) if isinstance(v, (int, float)) else str(v)
445
+ for k, v in DL_STATE.get('metrics', {}).items()},
446
+ })
447
+ elif MODEL_STATE:
448
+ info.update({
449
+ "model_name": MODEL_STATE.get('model_name', 'Unknown'),
450
+ "task_type": MODEL_STATE.get('task_type', 'classification'),
451
+ "target_column": TARGET_COLUMN,
452
+ "input_features": list(RAW_FEATURE_INFO.keys()),
453
+ "metrics": {k: float(v) if isinstance(v, (int, float)) else str(v)
454
+ for k, v in MODEL_STATE.get('metrics', {}).items()},
455
+ })
456
+ if DL_STATE:
457
+ info["dl_model"] = {
458
+ "algorithm": DL_STATE.get('algorithm', 'MLP'),
459
+ "task_type": DL_STATE.get('task_type', 'classification'),
460
+ "metrics": {k: float(v) if isinstance(v, (int, float)) else str(v)
461
+ for k, v in DL_STATE.get('metrics', {}).items()},
462
+ }
463
+ return jsonify(info)
464
+
465
+
466
+ @app.route('/predict', methods=['GET', 'POST'])
467
+ def predict():
468
+ if MODEL_STATE is None and DL_STATE is None:
469
+ return jsonify({"error": "No model loaded"}), 500
470
+
471
+ # GET request: show web form for interactive prediction
472
+ if request.method == 'GET':
473
+ if BEST_MODE == 'deep_learning' and DL_STATE:
474
+ model_name = DL_STATE.get('algorithm', 'MLP Neural Network')
475
+ task_type_val = DL_STATE.get('task_type', 'classification')
476
+ metrics = DL_STATE.get('metrics', {})
477
+ elif MODEL_STATE:
478
+ model_name = MODEL_STATE.get('model_name', 'Unknown')
479
+ task_type_val = MODEL_STATE.get('task_type', 'classification')
480
+ metrics = MODEL_STATE.get('metrics', {})
481
+ else:
482
+ model_name = 'Unknown'
483
+ task_type_val = 'classification'
484
+ metrics = {}
485
+
486
+ # Build form from RAW features (original column names user knows)
487
+ form_fields = ""
488
+ for col, info in RAW_FEATURE_INFO.items():
489
+ ftype = info.get("type", "numeric")
490
+ if ftype == "categorical":
491
+ options = info.get("options", [])
492
+ opts_html = "".join('<option value="' + str(o) + '">' + str(o) + '</option>' for o in options[:20])
493
+ form_fields += '<div style="margin:6px 0"><label style="display:inline-block;width:260px;font-weight:500">' + col + '</label><select name="' + col + '" style="padding:6px 10px;border:1px solid #d1d5db;border-radius:6px;width:214px">' + opts_html + '</select></div>' + chr(10)
494
+ else:
495
+ default_val = str(info.get("default", 0))
496
+ mn = str(info.get("min", ""))
497
+ mx = str(info.get("max", ""))
498
+ placeholder = (mn + " - " + mx) if mn != "" else ""
499
+ form_fields += '<div style="margin:6px 0"><label style="display:inline-block;width:260px;font-weight:500">' + col + '</label><input name="' + col + '" type="text" value="' + default_val + '" placeholder="' + placeholder + '" style="padding:6px 10px;border:1px solid #d1d5db;border-radius:6px;width:200px"></div>' + chr(10)
500
+
501
+ metrics_html = ""
502
+ for k, v in metrics.items():
503
+ if isinstance(v, float):
504
+ metrics_html += f"<li><b>{k}</b>: {v:.4f}</li>"
505
+
506
+ html_tpl = '<!DOCTYPE html>'
507
+ html_tpl += '<html><head><title>DataVision ML API</title>'
508
+ html_tpl += '<style>'
509
+ html_tpl += "body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; max-width: 800px; margin: 40px auto; padding: 0 20px; background: #f9fafb; color: #111827; }}"
510
+ html_tpl += 'h1 {{ color: #4F46E5; }} h2 {{ color: #1f2937; border-bottom: 2px solid #e5e7eb; padding-bottom: 8px; }}'
511
+ html_tpl += '.card {{ background: white; border-radius: 12px; padding: 24px; margin: 16px 0; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }}'
512
+ html_tpl += 'button {{ background: #4F46E5; color: white; border: none; padding: 12px 32px; border-radius: 8px; font-size: 16px; cursor: pointer; margin-top: 12px; }}'
513
+ html_tpl += 'button:hover {{ background: #4338CA; }}'
514
+ html_tpl += '#result {{ margin-top: 20px; padding: 16px; background: #f0fdf4; border-radius: 8px; display: none; }}'
515
+ html_tpl += '</style></head><body>'
516
+ html_tpl += '<h1>🚀 DataVision ML API</h1>'
517
+ html_tpl += '<div class="card"><h2>Model Info</h2>'
518
+ html_tpl += '<p>🏆 <b>Model:</b> ' + str(model_name) + ' | 🎯 <b>Target:</b> price_range | 📊 <b>Task:</b> ' + str(task_type_val) + ' | 📁 <b>Features:</b> ' + str(len(RAW_FEATURE_INFO)) + '</p>'
519
+ html_tpl += '<ul>' + metrics_html + '</ul></div>'
520
+ html_tpl += '<div class="card"><h2>🔮 Make a Prediction</h2>'
521
+ html_tpl += '<form id="predForm">' + form_fields + '<button type="submit">Predict</button></form>'
522
+ html_tpl += '<div id="result"></div></div>'
523
+ html_tpl += '<div class="card"><h2>📡 API Usage</h2>'
524
+ html_tpl += '<pre style="background:#1f2937;color:#e5e7eb;padding:16px;border-radius:8px;overflow-x:auto">'
525
+ html_tpl += 'POST /predict\nContent-Type: application/json\n\n' + chr(123) + '"feature1": val, "feature2": val' + chr(125)
526
+ html_tpl += '\n\nGET /health — Health check\nGET /model-info — Model metadata</pre></div>'
527
+ html_tpl += '<script>'
528
+ html_tpl += 'document.getElementById("predForm").onsubmit=function(e)' + chr(123)
529
+ html_tpl += 'e.preventDefault();var fd=new FormData(this);var data=' + chr(123) + chr(125) + ';'
530
+ html_tpl += 'fd.forEach(function(v,k)' + chr(123) + 'data[k]=isNaN(v)||v===""?v:parseFloat(v)' + chr(125) + ');'
531
+ html_tpl += 'fetch("/predict",' + chr(123) + 'method:"POST",headers:' + chr(123) + '"Content-Type":"application/json"' + chr(125) + ',body:JSON.stringify(data)' + chr(125) + ')'
532
+ html_tpl += '.then(function(r)' + chr(123) + 'return r.json()' + chr(125) + ')'
533
+ html_tpl += '.then(function(j)' + chr(123)
534
+ html_tpl += 'var h="<h3>🎯 Prediction: "+j.prediction+"</h3>";'
535
+ html_tpl += 'if(j.confidence)h+="<p>📊 Confidence: "+(j.confidence*100).toFixed(1)+"%</p>";'
536
+ html_tpl += 'if(j.probabilities)' + chr(123) + 'h+="<p>Probabilities:</p><ul>";'
537
+ html_tpl += 'var ent=Object.entries(j.probabilities).sort(function(a,b)' + chr(123) + 'return b[1]-a[1]' + chr(125) + ');'
538
+ html_tpl += 'for(var i=0;i<ent.length;i++)h+="<li><b>"+ent[i][0]+"</b>: "+(ent[i][1]*100).toFixed(1)+"%</li>";'
539
+ html_tpl += 'h+="</ul>"' + chr(125)
540
+ html_tpl += 'document.getElementById("result").innerHTML=h;document.getElementById("result").style.display="block"'
541
+ html_tpl += chr(125) + ')' + chr(125) + ';'
542
+ html_tpl += '</script></body></html>'
543
+ return html_tpl
544
+
545
+ # POST request: JSON API prediction
546
+ try:
547
+ data = request.get_json()
548
+ if not data:
549
+ return jsonify({"error": "No JSON body. Send features as JSON.", "expected_features": list(RAW_FEATURE_INFO.keys())}), 400
550
+
551
+ # Check if caller wants DL mode, or if BEST_MODE is deep_learning, or if only DL is available
552
+ req_mode = data.pop('_mode', None)
553
+ use_dl = (req_mode == 'dl') or (req_mode != 'ml' and BEST_MODE == 'deep_learning' and DL_STATE is not None) or (MODEL_STATE is None and DL_STATE is not None)
554
+
555
+ if use_dl and DL_STATE:
556
+ # Deep Learning prediction
557
+ X = preprocess_dl_input(data)
558
+ dl_model = DL_STATE['model']
559
+ label_encoder = DL_STATE.get('label_encoder')
560
+ task_type = DL_STATE.get('task_type', 'classification')
561
+ classes = DL_STATE.get('classes', [])
562
+
563
+ pred = dl_model.predict(X)[0]
564
+ result = {"raw_prediction": int(pred) if isinstance(pred, (np.integer,)) else float(pred)}
565
+
566
+ if 'classif' in task_type.lower() and label_encoder:
567
+ try:
568
+ result['prediction'] = str(label_encoder.inverse_transform([int(pred)])[0])
569
+ except:
570
+ result['prediction'] = str(pred)
571
+ else:
572
+ result['prediction'] = float(pred)
573
+
574
+ if 'classif' in task_type.lower() and hasattr(dl_model, 'predict_proba'):
575
+ try:
576
+ proba = dl_model.predict_proba(X)[0]
577
+ result['confidence'] = float(max(proba))
578
+ if classes:
579
+ result['probabilities'] = {
580
+ str(c): round(float(p), 4) for c, p in zip(classes, proba)
581
+ }
582
+ except:
583
+ pass
584
+
585
+ result['mode'] = 'deep_learning'
586
+ return jsonify(result)
587
+
588
+ # Traditional/NLP prediction
589
+ model = MODEL_STATE['model']
590
+ target_encoder = MODEL_STATE.get('target_encoder')
591
+ task_type = MODEL_STATE.get('task_type', 'classification')
592
+
593
+ X = preprocess_input(data)
594
+
595
+ prediction = model.predict(X)[0]
596
+ result = {"raw_prediction": int(prediction) if isinstance(prediction, (np.integer,)) else float(prediction)}
597
+
598
+ if 'classif' in task_type.lower() and target_encoder:
599
+ try:
600
+ result['prediction'] = str(target_encoder.inverse_transform([int(prediction)])[0])
601
+ except:
602
+ result['prediction'] = str(prediction)
603
+ else:
604
+ result['prediction'] = float(prediction)
605
+
606
+ if 'classif' in task_type.lower() and hasattr(model, 'predict_proba'):
607
+ try:
608
+ proba = model.predict_proba(X)[0]
609
+ result['confidence'] = float(max(proba))
610
+ if target_encoder:
611
+ result['probabilities'] = {
612
+ str(c): round(float(p), 4) for c, p in zip(target_encoder.classes_, proba)
613
+ }
614
+ except:
615
+ pass
616
+
617
+ return jsonify(result)
618
+ except Exception as e:
619
+ return jsonify({"error": str(e)}), 500
620
+
621
+
622
+ if __name__ == '__main__':
623
+ parser = argparse.ArgumentParser()
624
+ parser.add_argument('--port', type=int, default=5000)
625
+ parser.add_argument('--host', type=str, default='0.0.0.0')
626
+ args = parser.parse_args()
627
+
628
+ if load_model():
629
+ print(f"\n🌐 API Server on http://{args.host}:{args.port}")
630
+ print(f" POST /predict — Send JSON with your feature values")
631
+ print(f" GET /health — Health check")
632
+ print(f" GET /model-info — Your model details and expected features")
633
+ app.run(host=args.host, port=args.port, debug=False)
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/config.json ADDED
@@ -0,0 +1,615 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "target_column": "price_range",
3
+ "feature_columns": [
4
+ "battery_power",
5
+ "blue",
6
+ "clock_speed",
7
+ "dual_sim",
8
+ "fc",
9
+ "four_g",
10
+ "int_memory",
11
+ "m_dep",
12
+ "mobile_wt",
13
+ "n_cores",
14
+ "pc",
15
+ "px_height",
16
+ "px_width",
17
+ "ram",
18
+ "sc_h",
19
+ "sc_w",
20
+ "talk_time",
21
+ "three_g",
22
+ "touch_screen",
23
+ "wifi"
24
+ ],
25
+ "task_type": "classification",
26
+ "best_mode": "traditional",
27
+ "modes_trained": [
28
+ "traditional"
29
+ ],
30
+ "best_model_name": "RandomForest",
31
+ "metrics": {
32
+ "accuracy": 0.8925,
33
+ "f1": 0.892925039212415,
34
+ "precision": 0.8937266322786124,
35
+ "recall": 0.8925,
36
+ "roc_auc": 0.9796333333333334
37
+ },
38
+ "data_summary": {
39
+ "rows": 2000,
40
+ "columns": 21,
41
+ "features_engineered": 20
42
+ },
43
+ "feature_metadata": [
44
+ {
45
+ "name": "battery_power",
46
+ "type": "numeric",
47
+ "min": 510.0,
48
+ "max": 1987.0,
49
+ "mean": 1238.513
50
+ },
51
+ {
52
+ "name": "blue",
53
+ "type": "numeric",
54
+ "min": 0.0,
55
+ "max": 1.0,
56
+ "mean": 0.495
57
+ },
58
+ {
59
+ "name": "clock_speed",
60
+ "type": "numeric",
61
+ "min": 0.5,
62
+ "max": 3.0,
63
+ "mean": 1.52225
64
+ },
65
+ {
66
+ "name": "dual_sim",
67
+ "type": "numeric",
68
+ "min": 0.0,
69
+ "max": 1.0,
70
+ "mean": 0.5095
71
+ },
72
+ {
73
+ "name": "fc",
74
+ "type": "numeric",
75
+ "min": 0.0,
76
+ "max": 16.0,
77
+ "mean": 4.294
78
+ },
79
+ {
80
+ "name": "four_g",
81
+ "type": "numeric",
82
+ "min": 0.0,
83
+ "max": 1.0,
84
+ "mean": 0.5215
85
+ },
86
+ {
87
+ "name": "int_memory",
88
+ "type": "numeric",
89
+ "min": 2.0,
90
+ "max": 64.0,
91
+ "mean": 32.0465
92
+ },
93
+ {
94
+ "name": "m_dep",
95
+ "type": "numeric",
96
+ "min": 0.1,
97
+ "max": 1.0,
98
+ "mean": 0.50175
99
+ },
100
+ {
101
+ "name": "mobile_wt",
102
+ "type": "numeric",
103
+ "min": 80.0,
104
+ "max": 199.0,
105
+ "mean": 140.2395
106
+ },
107
+ {
108
+ "name": "n_cores",
109
+ "type": "numeric",
110
+ "min": 1.0,
111
+ "max": 8.0,
112
+ "mean": 4.5205
113
+ },
114
+ {
115
+ "name": "pc",
116
+ "type": "numeric",
117
+ "min": 0.0,
118
+ "max": 20.0,
119
+ "mean": 9.9165
120
+ },
121
+ {
122
+ "name": "px_height",
123
+ "type": "numeric",
124
+ "min": 15.0,
125
+ "max": 1791.01,
126
+ "mean": 644.4581000000001
127
+ },
128
+ {
129
+ "name": "ram",
130
+ "type": "numeric",
131
+ "min": 296.99,
132
+ "max": 3958.01,
133
+ "mean": 2124.2565
134
+ },
135
+ {
136
+ "name": "sc_h",
137
+ "type": "numeric",
138
+ "min": 5.0,
139
+ "max": 19.0,
140
+ "mean": 12.3065
141
+ },
142
+ {
143
+ "name": "sc_w",
144
+ "type": "numeric",
145
+ "min": 0.0,
146
+ "max": 17.0,
147
+ "mean": 5.763
148
+ },
149
+ {
150
+ "name": "three_g",
151
+ "type": "numeric",
152
+ "min": 1.0,
153
+ "max": 1.0,
154
+ "mean": 1.0
155
+ },
156
+ {
157
+ "name": "touch_screen",
158
+ "type": "numeric",
159
+ "min": 0.0,
160
+ "max": 1.0,
161
+ "mean": 0.503
162
+ },
163
+ {
164
+ "name": "wifi",
165
+ "type": "numeric",
166
+ "min": 0.0,
167
+ "max": 1.0,
168
+ "mean": 0.507
169
+ },
170
+ {
171
+ "name": "talk_time",
172
+ "type": "datetime",
173
+ "min": "1970-01-01T00:00:00.000000002",
174
+ "max": "1970-01-01T00:00:00.000000020",
175
+ "placeholder": "Select date..."
176
+ }
177
+ ],
178
+ "numeric_features": [
179
+ {
180
+ "name": "battery_power",
181
+ "type": "numeric",
182
+ "min": 510.0,
183
+ "max": 1987.0,
184
+ "mean": 1238.513
185
+ },
186
+ {
187
+ "name": "blue",
188
+ "type": "numeric",
189
+ "min": 0.0,
190
+ "max": 1.0,
191
+ "mean": 0.495
192
+ },
193
+ {
194
+ "name": "clock_speed",
195
+ "type": "numeric",
196
+ "min": 0.5,
197
+ "max": 3.0,
198
+ "mean": 1.52225
199
+ },
200
+ {
201
+ "name": "dual_sim",
202
+ "type": "numeric",
203
+ "min": 0.0,
204
+ "max": 1.0,
205
+ "mean": 0.5095
206
+ },
207
+ {
208
+ "name": "fc",
209
+ "type": "numeric",
210
+ "min": 0.0,
211
+ "max": 16.0,
212
+ "mean": 4.294
213
+ },
214
+ {
215
+ "name": "four_g",
216
+ "type": "numeric",
217
+ "min": 0.0,
218
+ "max": 1.0,
219
+ "mean": 0.5215
220
+ },
221
+ {
222
+ "name": "int_memory",
223
+ "type": "numeric",
224
+ "min": 2.0,
225
+ "max": 64.0,
226
+ "mean": 32.0465
227
+ },
228
+ {
229
+ "name": "m_dep",
230
+ "type": "numeric",
231
+ "min": 0.1,
232
+ "max": 1.0,
233
+ "mean": 0.50175
234
+ },
235
+ {
236
+ "name": "mobile_wt",
237
+ "type": "numeric",
238
+ "min": 80.0,
239
+ "max": 199.0,
240
+ "mean": 140.2395
241
+ },
242
+ {
243
+ "name": "n_cores",
244
+ "type": "numeric",
245
+ "min": 1.0,
246
+ "max": 8.0,
247
+ "mean": 4.5205
248
+ },
249
+ {
250
+ "name": "pc",
251
+ "type": "numeric",
252
+ "min": 0.0,
253
+ "max": 20.0,
254
+ "mean": 9.9165
255
+ },
256
+ {
257
+ "name": "px_height",
258
+ "type": "numeric",
259
+ "min": 15.0,
260
+ "max": 1791.01,
261
+ "mean": 644.4581000000001
262
+ },
263
+ {
264
+ "name": "ram",
265
+ "type": "numeric",
266
+ "min": 296.99,
267
+ "max": 3958.01,
268
+ "mean": 2124.2565
269
+ },
270
+ {
271
+ "name": "sc_h",
272
+ "type": "numeric",
273
+ "min": 5.0,
274
+ "max": 19.0,
275
+ "mean": 12.3065
276
+ },
277
+ {
278
+ "name": "sc_w",
279
+ "type": "numeric",
280
+ "min": 0.0,
281
+ "max": 17.0,
282
+ "mean": 5.763
283
+ },
284
+ {
285
+ "name": "three_g",
286
+ "type": "numeric",
287
+ "min": 1.0,
288
+ "max": 1.0,
289
+ "mean": 1.0
290
+ },
291
+ {
292
+ "name": "touch_screen",
293
+ "type": "numeric",
294
+ "min": 0.0,
295
+ "max": 1.0,
296
+ "mean": 0.503
297
+ },
298
+ {
299
+ "name": "wifi",
300
+ "type": "numeric",
301
+ "min": 0.0,
302
+ "max": 1.0,
303
+ "mean": 0.507
304
+ }
305
+ ],
306
+ "categorical_features": [],
307
+ "text_features": [],
308
+ "nlp_text_column": null,
309
+ "results_per_mode": {
310
+ "traditional": {
311
+ "success": true,
312
+ "best_model": "RandomForest",
313
+ "metrics": {
314
+ "accuracy": 0.8925,
315
+ "f1": 0.892925039212415,
316
+ "precision": 0.8937266322786124,
317
+ "recall": 0.8925,
318
+ "roc_auc": 0.9796333333333334
319
+ },
320
+ "leaderboard": [
321
+ {
322
+ "name": "RandomForest",
323
+ "metrics": {
324
+ "accuracy": 0.8925,
325
+ "f1": 0.892925039212415,
326
+ "precision": 0.8937266322786124,
327
+ "recall": 0.8925,
328
+ "roc_auc": 0.9796333333333334
329
+ },
330
+ "score": 0.8925
331
+ }
332
+ ],
333
+ "task_type": "classification",
334
+ "algorithms_used": [
335
+ "random_forest"
336
+ ],
337
+ "feature_columns": [
338
+ "battery_power",
339
+ "blue",
340
+ "clock_speed",
341
+ "dual_sim",
342
+ "fc",
343
+ "four_g",
344
+ "int_memory",
345
+ "m_dep",
346
+ "mobile_wt",
347
+ "n_cores",
348
+ "pc",
349
+ "px_height",
350
+ "px_width",
351
+ "ram",
352
+ "sc_h",
353
+ "sc_w",
354
+ "talk_time",
355
+ "three_g",
356
+ "touch_screen",
357
+ "wifi"
358
+ ],
359
+ "feature_importance": [
360
+ {
361
+ "feature": "sc_h",
362
+ "importance": 0.5233,
363
+ "rank": 1
364
+ },
365
+ {
366
+ "feature": "battery_power",
367
+ "importance": 0.0759,
368
+ "rank": 2
369
+ },
370
+ {
371
+ "feature": "ram",
372
+ "importance": 0.0568,
373
+ "rank": 3
374
+ },
375
+ {
376
+ "feature": "px_height",
377
+ "importance": 0.0535,
378
+ "rank": 4
379
+ },
380
+ {
381
+ "feature": "mobile_wt",
382
+ "importance": 0.0384,
383
+ "rank": 5
384
+ },
385
+ {
386
+ "feature": "int_memory",
387
+ "importance": 0.0339,
388
+ "rank": 6
389
+ },
390
+ {
391
+ "feature": "touch_screen",
392
+ "importance": 0.0278,
393
+ "rank": 7
394
+ },
395
+ {
396
+ "feature": "pc",
397
+ "importance": 0.0271,
398
+ "rank": 8
399
+ },
400
+ {
401
+ "feature": "three_g",
402
+ "importance": 0.0262,
403
+ "rank": 9
404
+ },
405
+ {
406
+ "feature": "clock_speed",
407
+ "importance": 0.0243,
408
+ "rank": 10
409
+ },
410
+ {
411
+ "feature": "sc_w",
412
+ "importance": 0.0236,
413
+ "rank": 11
414
+ },
415
+ {
416
+ "feature": "m_dep",
417
+ "importance": 0.0226,
418
+ "rank": 12
419
+ },
420
+ {
421
+ "feature": "fc",
422
+ "importance": 0.022,
423
+ "rank": 13
424
+ },
425
+ {
426
+ "feature": "n_cores",
427
+ "importance": 0.0209,
428
+ "rank": 14
429
+ },
430
+ {
431
+ "feature": "four_g",
432
+ "importance": 0.0064,
433
+ "rank": 15
434
+ }
435
+ ],
436
+ "feature_metadata": [
437
+ {
438
+ "name": "battery_power",
439
+ "type": "numeric",
440
+ "min": 510.0,
441
+ "max": 1987.0,
442
+ "mean": 1238.513
443
+ },
444
+ {
445
+ "name": "blue",
446
+ "type": "numeric",
447
+ "min": 0.0,
448
+ "max": 1.0,
449
+ "mean": 0.495
450
+ },
451
+ {
452
+ "name": "clock_speed",
453
+ "type": "numeric",
454
+ "min": 0.5,
455
+ "max": 3.0,
456
+ "mean": 1.52225
457
+ },
458
+ {
459
+ "name": "dual_sim",
460
+ "type": "numeric",
461
+ "min": 0.0,
462
+ "max": 1.0,
463
+ "mean": 0.5095
464
+ },
465
+ {
466
+ "name": "fc",
467
+ "type": "numeric",
468
+ "min": 0.0,
469
+ "max": 16.0,
470
+ "mean": 4.294
471
+ },
472
+ {
473
+ "name": "four_g",
474
+ "type": "numeric",
475
+ "min": 0.0,
476
+ "max": 1.0,
477
+ "mean": 0.5215
478
+ },
479
+ {
480
+ "name": "int_memory",
481
+ "type": "numeric",
482
+ "min": 2.0,
483
+ "max": 64.0,
484
+ "mean": 32.0465
485
+ },
486
+ {
487
+ "name": "m_dep",
488
+ "type": "numeric",
489
+ "min": 0.1,
490
+ "max": 1.0,
491
+ "mean": 0.50175
492
+ },
493
+ {
494
+ "name": "mobile_wt",
495
+ "type": "numeric",
496
+ "min": 80.0,
497
+ "max": 199.0,
498
+ "mean": 140.2395
499
+ },
500
+ {
501
+ "name": "n_cores",
502
+ "type": "numeric",
503
+ "min": 1.0,
504
+ "max": 8.0,
505
+ "mean": 4.5205
506
+ },
507
+ {
508
+ "name": "pc",
509
+ "type": "numeric",
510
+ "min": 0.0,
511
+ "max": 20.0,
512
+ "mean": 9.9165
513
+ },
514
+ {
515
+ "name": "px_height",
516
+ "type": "numeric",
517
+ "min": 15.0,
518
+ "max": 1791.01,
519
+ "mean": 644.4581000000001
520
+ },
521
+ {
522
+ "name": "ram",
523
+ "type": "numeric",
524
+ "min": 296.99,
525
+ "max": 3958.01,
526
+ "mean": 2124.2565
527
+ },
528
+ {
529
+ "name": "sc_h",
530
+ "type": "numeric",
531
+ "min": 5.0,
532
+ "max": 19.0,
533
+ "mean": 12.3065
534
+ },
535
+ {
536
+ "name": "sc_w",
537
+ "type": "numeric",
538
+ "min": 0.0,
539
+ "max": 17.0,
540
+ "mean": 5.763
541
+ },
542
+ {
543
+ "name": "three_g",
544
+ "type": "numeric",
545
+ "min": 1.0,
546
+ "max": 1.0,
547
+ "mean": 1.0
548
+ },
549
+ {
550
+ "name": "touch_screen",
551
+ "type": "numeric",
552
+ "min": 0.0,
553
+ "max": 1.0,
554
+ "mean": 0.503
555
+ },
556
+ {
557
+ "name": "wifi",
558
+ "type": "numeric",
559
+ "min": 0.0,
560
+ "max": 1.0,
561
+ "mean": 0.507
562
+ },
563
+ {
564
+ "name": "talk_time",
565
+ "type": "datetime",
566
+ "min": "1970-01-01T00:00:00.000000002",
567
+ "max": "1970-01-01T00:00:00.000000020",
568
+ "placeholder": "Select date..."
569
+ }
570
+ ],
571
+ "n_rows": 2000,
572
+ "n_cols": 21,
573
+ "cleaned_file_path": "cleaned_1782453484.csv",
574
+ "reliability_score": 75,
575
+ "validation_warnings": null
576
+ }
577
+ },
578
+ "leaderboard": [
579
+ {
580
+ "mode": "Traditional ML",
581
+ "model": "RandomForest",
582
+ "score": 0.8925,
583
+ "metrics": {
584
+ "accuracy": 0.8925,
585
+ "f1": 0.892925039212415,
586
+ "precision": 0.8937266322786124,
587
+ "recall": 0.8925,
588
+ "roc_auc": 0.9796333333333334
589
+ }
590
+ }
591
+ ],
592
+ "actual_model_params": {
593
+ "bootstrap": true,
594
+ "ccp_alpha": 0.0,
595
+ "class_weight": "balanced",
596
+ "criterion": "gini",
597
+ "max_depth": 10,
598
+ "max_features": "sqrt",
599
+ "max_leaf_nodes": null,
600
+ "max_samples": null,
601
+ "min_impurity_decrease": 0.0,
602
+ "min_samples_leaf": 1,
603
+ "min_samples_split": 2,
604
+ "min_weight_fraction_leaf": 0.0,
605
+ "monotonic_cst": null,
606
+ "n_estimators": 150,
607
+ "n_jobs": -1,
608
+ "oob_score": false,
609
+ "random_state": 42,
610
+ "verbose": 0,
611
+ "warm_start": false
612
+ },
613
+ "actual_model_class": "RandomForestClassifier",
614
+ "has_production_engineer": true
615
+ }
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/evaluate.py ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 📊 Model Evaluation Script — Shows YOUR AutoML Results
3
+ ========================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ ╔════════════════════════════════════════════════════════════════════╗
7
+ ║ This script shows the EXACT SAME metrics as your AutoML training. ║
8
+ ║ Supports Traditional ML, Deep Learning, and NLP models. ║
9
+ ╚════════════════════════════════════════════════════════════════════╝
10
+
11
+ Model : RandomForest
12
+ Target Column : price_range
13
+ Task Type : classification
14
+
15
+ YOUR AUTOML METRICS:
16
+ # accuracy: 0.8925
17
+ # f1: 0.8929
18
+ # precision: 0.8937
19
+ # recall: 0.8925
20
+ # roc_auc: 0.9796
21
+
22
+ Usage:
23
+ python evaluate.py # Show YOUR AutoML results
24
+ python evaluate.py --reeval # Re-evaluate with data (may differ)
25
+ """
26
+
27
+ import os
28
+ import pickle
29
+ import argparse
30
+ import warnings
31
+ import numpy as np
32
+ import pandas as pd
33
+ from sklearn.model_selection import train_test_split, cross_val_score
34
+ from sklearn.preprocessing import LabelEncoder, StandardScaler
35
+ from sklearn.metrics import (
36
+ accuracy_score, precision_score, recall_score, f1_score,
37
+ r2_score, mean_absolute_error, mean_squared_error,
38
+ confusion_matrix, classification_report
39
+ )
40
+ try:
41
+ from sklearn.metrics import roc_auc_score
42
+ except ImportError:
43
+ roc_auc_score = None
44
+
45
+ warnings.filterwarnings('ignore')
46
+
47
+ TARGET_COLUMN = 'price_range'
48
+ TASK_TYPE = 'classification'
49
+ BEST_MODE = 'traditional' # Which mode won during AutoML
50
+
51
+
52
+ def print_classification_metrics(y_test, y_pred, model=None, X_test=None, target_encoder=None):
53
+ """Print all classification metrics including ROC-AUC."""
54
+ acc = accuracy_score(y_test, y_pred)
55
+ prec = precision_score(y_test, y_pred, average='weighted', zero_division=0)
56
+ rec = recall_score(y_test, y_pred, average='weighted', zero_division=0)
57
+ f1 = f1_score(y_test, y_pred, average='weighted', zero_division=0)
58
+ f1_macro = f1_score(y_test, y_pred, average='macro', zero_division=0)
59
+
60
+ print(f"\n✅ ACCURACY: {acc:.4f} ({acc*100:.1f}%)")
61
+ print(f" Precision: {prec:.4f} ({prec*100:.1f}%)")
62
+ print(f" Recall: {rec:.4f} ({rec*100:.1f}%)")
63
+ print(f" F1-Weighted: {f1:.4f} ({f1*100:.1f}%)")
64
+ print(f" F1-Macro: {f1_macro:.4f} ({f1_macro*100:.1f}%)")
65
+
66
+ # ROC-AUC
67
+ if roc_auc_score is not None and model is not None and X_test is not None:
68
+ try:
69
+ n_classes = len(np.unique(y_test))
70
+ if n_classes == 2 and hasattr(model, 'predict_proba'):
71
+ y_proba = model.predict_proba(X_test)[:, 1]
72
+ roc = roc_auc_score(y_test, y_proba)
73
+ print(f" ROC-AUC: {roc:.4f} ({roc*100:.1f}%)")
74
+ elif n_classes > 2 and hasattr(model, 'predict_proba'):
75
+ y_proba = model.predict_proba(X_test)
76
+ roc = roc_auc_score(y_test, y_proba, multi_class='ovr', average='weighted')
77
+ print(f" ROC-AUC: {roc:.4f} ({roc*100:.1f}%)")
78
+ elif n_classes == 2 and hasattr(model, 'decision_function'):
79
+ y_scores = model.decision_function(X_test)
80
+ roc = roc_auc_score(y_test, y_scores)
81
+ print(f" ROC-AUC: {roc:.4f} ({roc*100:.1f}%)")
82
+ except Exception:
83
+ pass
84
+
85
+ print("\n📊 Classification Report:")
86
+ if target_encoder is not None:
87
+ try:
88
+ print(classification_report(y_test, y_pred, target_names=target_encoder.classes_))
89
+ except Exception:
90
+ print(classification_report(y_test, y_pred))
91
+ else:
92
+ print(classification_report(y_test, y_pred))
93
+
94
+ print("\n📊 Confusion Matrix:")
95
+ cm = confusion_matrix(y_test, y_pred)
96
+ print(cm)
97
+
98
+
99
+ def print_regression_metrics(y_test, y_pred):
100
+ """Print all regression metrics."""
101
+ r2 = r2_score(y_test, y_pred)
102
+ mae = mean_absolute_error(y_test, y_pred)
103
+ mse = mean_squared_error(y_test, y_pred)
104
+ rmse = np.sqrt(mse)
105
+
106
+ print(f"\n✅ R² SCORE: {r2:.4f}")
107
+ print(f" MSE: {mse:.4f}")
108
+ print(f" RMSE: {rmse:.4f}")
109
+ print(f" MAE: {mae:.4f}")
110
+
111
+
112
+ def load_dl_model(dl_path='deep_learning_model.pkl'):
113
+ """Load the Deep Learning model."""
114
+ if not os.path.exists(dl_path):
115
+ return None
116
+ try:
117
+ with open(dl_path, 'rb') as f:
118
+ state = pickle.load(f)
119
+ return state
120
+ except Exception as e:
121
+ print(f"⚠️ Failed to load DL model: {e}")
122
+ return None
123
+
124
+
125
+ def main():
126
+ parser = argparse.ArgumentParser(description="Show your AutoML training results")
127
+ parser.add_argument('--model', type=str, default='model.pkl')
128
+ parser.add_argument('--dl-model', type=str, default='deep_learning_model.pkl', help='Path to DL model file')
129
+ parser.add_argument('--mode', type=str, default='auto', choices=['auto', 'traditional', 'dl'], help='Evaluation mode')
130
+ parser.add_argument('--data', type=str, default='data/cleaned_data.csv')
131
+ parser.add_argument('--reeval', action='store_true', help='Re-evaluate with fresh train/test split (may differ from AutoML)')
132
+ parser.add_argument('--cv', type=int, default=5, help='Number of cross-validation folds for --reeval')
133
+ args = parser.parse_args()
134
+
135
+ print("\n" + "=" * 60)
136
+ print("📊 DataVision AutoML — Your Training Results")
137
+ print("=" * 60)
138
+
139
+ # Decide which model to load based on BEST_MODE
140
+ use_dl = False
141
+ state = None
142
+
143
+ if args.mode == 'dl' or (args.mode == 'auto' and BEST_MODE == 'deep_learning'):
144
+ state = load_dl_model(args.dl_model)
145
+ if state:
146
+ use_dl = True
147
+ else:
148
+ print("⚠️ DL model not found, falling back to traditional ML model...")
149
+
150
+ if not use_dl and args.mode != 'dl':
151
+ if os.path.exists(args.model):
152
+ with open(args.model, 'rb') as f:
153
+ state = pickle.load(f)
154
+ elif os.path.exists(args.dl_model):
155
+ state = load_dl_model(args.dl_model)
156
+ if state:
157
+ use_dl = True
158
+
159
+ if state is None:
160
+ print("❌ No model found. Provide model.pkl or deep_learning_model.pkl")
161
+ return
162
+
163
+ model = state['model']
164
+ task_type = state.get('task_type', TASK_TYPE)
165
+ is_clf = 'classif' in task_type.lower()
166
+ mode_display = 'deep_learning' if use_dl else state.get('mode', 'traditional')
167
+ model_name = state.get('model_name', state.get('algorithm', 'Unknown'))
168
+
169
+ print(f"\n🏆 Model: {model_name}")
170
+ print(f"🎯 Task: {task_type}")
171
+ print(f"📊 Mode: {mode_display}")
172
+ if use_dl:
173
+ print(f"🧠 Architecture: {state.get('algorithm', 'MLP')}")
174
+ print(f" Features: {len(state.get('feature_columns', state.get('feature_metadata', [])))}")
175
+
176
+ # =========================================================================
177
+ # OPTION 1: Use SAVED y_test/y_pred (EXACT same as AutoML showed)
178
+ # =========================================================================
179
+ y_test_saved = state.get('y_test')
180
+ y_pred_saved = state.get('y_pred')
181
+
182
+ if not args.reeval and y_test_saved is not None and y_pred_saved is not None:
183
+ print("\n" + "=" * 60)
184
+ print("📋 YOUR AUTOML TRAINING RESULTS (from saved predictions)")
185
+ print("=" * 60)
186
+
187
+ y_test = np.array(y_test_saved)
188
+ y_pred = np.array(y_pred_saved)
189
+
190
+ # For ROC-AUC we need X_test and the model — try to reconstruct
191
+ X_test_for_roc = None
192
+ if use_dl:
193
+ X_test_saved = state.get('X_test')
194
+ if X_test_saved is not None:
195
+ X_test_for_roc = np.array(X_test_saved)
196
+
197
+ if is_clf:
198
+ print_classification_metrics(
199
+ y_test, y_pred, model=model,
200
+ X_test=X_test_for_roc,
201
+ target_encoder=state.get('target_encoder') if not use_dl else state.get('label_encoder')
202
+ )
203
+ else:
204
+ print_regression_metrics(y_test, y_pred)
205
+
206
+ # Also show stored metrics
207
+ stored_metrics = state.get('metrics', {})
208
+ if stored_metrics:
209
+ print("\n📋 Stored Metrics from Training:")
210
+ for k, v in stored_metrics.items():
211
+ if isinstance(v, float):
212
+ print(f" {k}: {v:.4f}")
213
+ else:
214
+ print(f" {k}: {v}")
215
+
216
+ print("\n" + "=" * 60)
217
+ print("✅ These are your EXACT AutoML results.")
218
+ print(" Use --reeval to re-evaluate with fresh data split (may differ).")
219
+ print("=" * 60)
220
+ return
221
+
222
+ # =========================================================================
223
+ # OPTION 2: Re-evaluate (may produce different results due to random split)
224
+ # =========================================================================
225
+ print("\n⚠️ Re-evaluating with fresh train/test split...")
226
+ print(" Note: Results may differ from AutoML due to different random split.")
227
+
228
+ if not os.path.exists(args.data):
229
+ print(f"⚠️ Data file not found: {args.data}")
230
+ print(" Showing stored metrics only:")
231
+ for k, v in state.get('metrics', {}).items():
232
+ print(f" {k}: {v}")
233
+ return
234
+
235
+ df = pd.read_csv(args.data)
236
+ target_col = state.get('target_column', TARGET_COLUMN)
237
+ feature_cols = state.get('feature_columns', [])
238
+
239
+ if target_col not in df.columns:
240
+ print(f"❌ Target '{target_col}' not in data")
241
+ return
242
+
243
+ df = df.dropna(subset=[target_col])
244
+
245
+ if use_dl:
246
+ # =====================================================================
247
+ # DL Re-evaluation: Use the DL model's preprocessing pipeline
248
+ # =====================================================================
249
+ feature_metadata = state.get('feature_metadata', [])
250
+ scaler = state.get('scaler')
251
+ label_encoder = state.get('label_encoder')
252
+
253
+ # Build feature matrix from metadata
254
+ X_parts = []
255
+ for meta in feature_metadata:
256
+ col = meta['name']
257
+ if meta.get('type') == 'numeric':
258
+ if col in df.columns:
259
+ X_parts.append(df[col].apply(pd.to_numeric, errors='coerce').fillna(meta.get('mean', 0)).values.reshape(-1, 1))
260
+ else:
261
+ X_parts.append(np.full((len(df), 1), meta.get('mean', 0)))
262
+ elif meta.get('type') == 'categorical':
263
+ if col in df.columns:
264
+ encoding = meta.get('encoding', {})
265
+ X_parts.append(df[col].astype(str).map(lambda v: encoding.get(v, 0)).values.reshape(-1, 1))
266
+ else:
267
+ X_parts.append(np.zeros((len(df), 1)))
268
+
269
+ if X_parts:
270
+ X = np.hstack(X_parts).astype(float)
271
+ else:
272
+ X = df.drop(columns=[target_col]).select_dtypes(include=[np.number]).fillna(0).values
273
+
274
+ y = df[target_col].values
275
+ if is_clf and label_encoder:
276
+ try:
277
+ y = label_encoder.transform(y)
278
+ except Exception:
279
+ le = LabelEncoder()
280
+ y = le.fit_transform(y)
281
+
282
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
283
+
284
+ # Scale AFTER split (same as training)
285
+ if scaler:
286
+ X_train = scaler.transform(X_train)
287
+ X_test = scaler.transform(X_test)
288
+
289
+ y_pred = model.predict(X_test)
290
+
291
+ if is_clf:
292
+ print(f"\n📊 Deep Learning Classification Metrics (re-evaluated):")
293
+ print_classification_metrics(y_test, y_pred, model=model, X_test=X_test,
294
+ target_encoder=label_encoder)
295
+ else:
296
+ print(f"\n📊 Deep Learning Regression Metrics (re-evaluated):")
297
+ print_regression_metrics(y_test, y_pred)
298
+ else:
299
+ # =====================================================================
300
+ # Traditional ML Re-evaluation
301
+ # =====================================================================
302
+ X = df.drop(columns=[target_col])
303
+ y = df[target_col]
304
+
305
+ target_encoder = state.get('target_encoder')
306
+ if is_clf and target_encoder:
307
+ y = pd.Series(target_encoder.transform(y))
308
+
309
+ label_encoders = state.get('label_encoders', {})
310
+ for col in X.select_dtypes(include=['object', 'category']).columns:
311
+ if col in label_encoders:
312
+ try:
313
+ X[col] = label_encoders[col].transform(X[col].astype(str))
314
+ except:
315
+ le = LabelEncoder()
316
+ X[col] = le.fit_transform(X[col].astype(str))
317
+ else:
318
+ le = LabelEncoder()
319
+ X[col] = le.fit_transform(X[col].astype(str))
320
+
321
+ X = X.fillna(0)
322
+
323
+ scaler = state.get('scaler')
324
+ if scaler and feature_cols:
325
+ avail = [c for c in feature_cols if c in X.columns]
326
+ if avail:
327
+ X = X[avail]
328
+ X = pd.DataFrame(scaler.transform(X), columns=avail)
329
+
330
+ # Align columns with model's expected features
331
+ if feature_cols:
332
+ for col in feature_cols:
333
+ if col not in X.columns:
334
+ X[col] = 0
335
+ X = X.reindex(columns=feature_cols, fill_value=0)
336
+
337
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
338
+
339
+ y_pred = model.predict(X_test)
340
+ y_pred_train = model.predict(X_train)
341
+
342
+ if is_clf:
343
+ print(f"\n📊 Classification Metrics (re-evaluated):")
344
+ print_classification_metrics(y_test, y_pred, model=model, X_test=X_test,
345
+ target_encoder=state.get('target_encoder'))
346
+
347
+ # Overfitting check
348
+ train_acc = accuracy_score(y_train, y_pred_train)
349
+ test_acc = accuracy_score(y_test, y_pred)
350
+ gap = train_acc - test_acc
351
+ if gap > 0.1:
352
+ print(f"\n⚠️ Overfitting detected (train-test gap: {gap:.4f})")
353
+ elif gap < 0.01:
354
+ print(f"\n✅ Good generalization (train-test gap: {gap:.4f})")
355
+
356
+ try:
357
+ cv_scores = cross_val_score(model, X, y, cv=args.cv, scoring='accuracy')
358
+ print(f"\n🔄 {args.cv}-Fold CV Accuracy: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")
359
+ except Exception as e:
360
+ print(f" CV failed: {e}")
361
+ else:
362
+ print(f"\n📊 Regression Metrics (re-evaluated):")
363
+ print_regression_metrics(y_test, y_pred)
364
+
365
+ try:
366
+ cv_scores = cross_val_score(model, X, y, cv=args.cv, scoring='r2')
367
+ print(f"\n🔄 {args.cv}-Fold CV R²: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")
368
+ except:
369
+ pass
370
+
371
+ print("\n✅ Evaluation complete!")
372
+
373
+
374
+ if __name__ == '__main__':
375
+ main()
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/predict.py ADDED
@@ -0,0 +1,728 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🔮 Prediction Script — Uses Your ACTUAL Trained Model
3
+ ========================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ ╔════════════════════════════════════════════════════════════════════╗
7
+ ║ This script loads YOUR model.pkl that was trained by AutoML. ║
8
+ ║ The predictions match EXACTLY what you saw in DataVision. ║
9
+ ╚════════════════════════════════════════════════════════════════════╝
10
+
11
+ YOUR MODEL:
12
+ 🏆 Model : RandomForest
13
+ 🎯 Target : price_range
14
+ 📊 Task : classification
15
+ 📁 Features : 20 columns
16
+
17
+ Usage:
18
+ python predict.py # Interactive mode
19
+ python predict.py --json '{"col1": 1, "col2": "A"}' # JSON input
20
+ python predict.py --csv input.csv # Batch predictions
21
+ """
22
+
23
+ import os
24
+ import pickle
25
+ import argparse
26
+ import json
27
+ import re
28
+ import warnings
29
+ import numpy as np
30
+ import pandas as pd
31
+ from sklearn.preprocessing import LabelEncoder, RobustScaler
32
+
33
+ warnings.filterwarnings('ignore')
34
+
35
+ # Your actual feature columns from training
36
+ FEATURE_COLUMNS = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt', 'n_cores', 'pc', 'px_height', 'px_width', 'ram', 'sc_h', 'sc_w', 'talk_time', 'three_g', 'touch_screen', 'wifi']
37
+ TARGET_COLUMN = 'price_range'
38
+ BEST_MODE = 'traditional' # Which mode won during AutoML
39
+
40
+ # Feature metadata with actual types and ranges from your dataset
41
+ FEATURE_INFO = {
42
+ "battery_power": {"type": "numeric", "min": 510.0, "max": 1987.0, "default": 1238.51},
43
+ "blue": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.49},
44
+ "clock_speed": {"type": "numeric", "min": 0.5, "max": 3.0, "default": 1.52},
45
+ "dual_sim": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
46
+ "fc": {"type": "numeric", "min": 0.0, "max": 16.0, "default": 4.29},
47
+ "four_g": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.52},
48
+ "int_memory": {"type": "numeric", "min": 2.0, "max": 64.0, "default": 32.05},
49
+ "m_dep": {"type": "numeric", "min": 0.1, "max": 1.0, "default": 0.50},
50
+ "mobile_wt": {"type": "numeric", "min": 80.0, "max": 199.0, "default": 140.24},
51
+ "n_cores": {"type": "numeric", "min": 1.0, "max": 8.0, "default": 4.52},
52
+ "pc": {"type": "numeric", "min": 0.0, "max": 20.0, "default": 9.92},
53
+ "px_height": {"type": "numeric", "min": 15.0, "max": 1791.01, "default": 644.46},
54
+ "ram": {"type": "numeric", "min": 296.99, "max": 3958.01, "default": 2124.26},
55
+ "sc_h": {"type": "numeric", "min": 5.0, "max": 19.0, "default": 12.31},
56
+ "sc_w": {"type": "numeric", "min": 0.0, "max": 17.0, "default": 5.76},
57
+ "three_g": {"type": "numeric", "min": 1.0, "max": 1.0, "default": 1.00},
58
+ "touch_screen": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.50},
59
+ "wifi": {"type": "numeric", "min": 0.0, "max": 1.0, "default": 0.51},
60
+ }
61
+
62
+
63
+ # =============================================================================
64
+ # Standalone Feature Engineer — Replicates DataVision's EXACT preprocessing
65
+ # =============================================================================
66
+ class StandaloneFeatureEngineer:
67
+ """Replicate DataVision's ProductionFeatureEngineer.transform() pipeline.
68
+
69
+ Loaded from feature_engineer.pkl which contains all fitted sklearn
70
+ transformers, encoders, and feature names from your AutoML training.
71
+ """
72
+
73
+ def __init__(self, state):
74
+ self.transformers = state.get('transformers', {})
75
+ self.encoders = state.get('encoders', {})
76
+ self.feature_names = state.get('feature_names', [])
77
+ self.original_columns = state.get('original_columns', [])
78
+ self.selected_feature_indices = state.get('selected_feature_indices', None)
79
+ self.is_fitted = state.get('is_fitted', True)
80
+ # Stopwords / sentiment words for text features
81
+ self.stopwords = set("i me my myself we our ours ourselves you your yours yourself yourselves he him his himself she her hers herself it its itself they them their theirs themselves what which who whom this that these those am is are was were be been being have has had having do does did doing a an the and but if or because as until while of at by for with about against between through during before after above below to from up down in out on off over under again further then once here there when where why how all both each few more most other some such no nor not only own same so than too very s t can will just don should now d ll m o re ve y ain aren couldn didn doesn hadn hasn haven isn ma mightn mustn needn shan shouldn wasn weren won wouldn".split())
82
+ self.positive_words = set("good great excellent amazing wonderful fantastic brilliant outstanding superb love best perfect beautiful awesome incredible impressive magnificent exceptional fabulous terrific marvelous stellar remarkable extraordinary delightful gorgeous splendid spectacular phenomenal exquisite lovely charming pleasant nice fine happy joy enjoy satisfied pleased glad cheerful excited thrilled".split())
83
+ self.negative_words = set("bad terrible horrible awful worst poor disappointing disgusting pathetic useless dreadful atrocious appalling abysmal mediocre inferior unpleasant nasty vile wretched miserable lousy horrendous hideous ghastly hideous deplorable lamentable pitiful woeful tragic sad angry frustrated annoyed irritated disappointed upset unhappy distressed sorrowful gloomy depressed".split())
84
+
85
+ def _clean_text(self, text):
86
+ text = str(text).lower()
87
+ text = re.sub(r'<[^>]+>', ' ', text)
88
+ text = re.sub(r'http\S+', ' ', text)
89
+ text = re.sub(r'[^a-z0-9\s]', ' ', text)
90
+ text = ' '.join(w for w in text.split() if w not in self.stopwords and len(w) > 1)
91
+ return text if text.strip() else 'empty'
92
+
93
+ def _text_statistics(self, text):
94
+ text = str(text)
95
+ words = text.split()
96
+ word_lengths = [len(w) for w in words] if words else [0]
97
+ unique_ratio = len(set(words)) / max(len(words), 1)
98
+ features = [
99
+ len(text), len(words), np.mean(word_lengths), np.std(word_lengths),
100
+ np.max(word_lengths) if word_lengths else 0, unique_ratio,
101
+ text.count('!'), text.count('?'),
102
+ sum(1 for c in text if c.isupper()) / max(len(text), 1),
103
+ len([w for w in words if len(w) > 6]) / max(len(words), 1),
104
+ ]
105
+ return features
106
+
107
+ def _sentiment_score(self, text):
108
+ words = set(str(text).lower().split())
109
+ pos = len(words & self.positive_words)
110
+ neg = len(words & self.negative_words)
111
+ total = max(pos + neg, 1)
112
+ return [pos / total, neg / total, (pos - neg) / total, total]
113
+
114
+ def transform(self, df):
115
+ if not self.is_fitted:
116
+ raise ValueError("Feature engineer not fitted!")
117
+ df = df.copy()
118
+
119
+ # Handle duplicate column names
120
+ cols = df.columns.tolist()
121
+ seen = {}
122
+ new_cols = []
123
+ for col in cols:
124
+ if col in seen:
125
+ seen[col] += 1
126
+ new_cols.append(f"{col}_{seen[col]}")
127
+ else:
128
+ seen[col] = 0
129
+ new_cols.append(col)
130
+ df.columns = new_cols
131
+
132
+ feature_parts = []
133
+
134
+ # ===== NUMERIC FEATURES =====
135
+ if 'numeric_cols' in self.transformers:
136
+ numeric_cols = self.transformers['numeric_cols']
137
+ for col in numeric_cols:
138
+ if col not in df.columns:
139
+ df[col] = 0
140
+ try:
141
+ numeric_data = df[numeric_cols].apply(pd.to_numeric, errors='coerce').fillna(0).values
142
+ scaler = self.transformers['numeric_scaler']
143
+ numeric_scaled = scaler.transform(numeric_data)
144
+ numeric_scaled = np.nan_to_num(numeric_scaled, nan=0.0, posinf=0.0, neginf=0.0)
145
+ feature_parts.append(numeric_scaled)
146
+
147
+ # Polynomial features
148
+ if 2 <= len(numeric_cols) <= 10:
149
+ interactions = []
150
+ for i in range(min(len(numeric_cols), 4)):
151
+ interactions.append((numeric_scaled[:, i] ** 2).reshape(-1, 1))
152
+ for i in range(min(len(numeric_cols), 4)):
153
+ for j in range(i + 1, min(len(numeric_cols), 4)):
154
+ interactions.append((numeric_scaled[:, i] * numeric_scaled[:, j]).reshape(-1, 1))
155
+ if interactions:
156
+ feature_parts.append(np.hstack(interactions))
157
+ except Exception as e:
158
+ print(f" Warning: Numeric transform error: {e}")
159
+ feature_parts.append(np.zeros((len(df), len(numeric_cols))))
160
+
161
+ # ===== CATEGORICAL: One-hot =====
162
+ for key, cols in self.transformers.items():
163
+ if key.endswith('_onehot'):
164
+ col_name = key.replace('_onehot', '')
165
+ try:
166
+ if col_name in df.columns:
167
+ series = df[col_name].fillna('_MISSING_').astype(str).str.strip()
168
+ dummies = pd.get_dummies(series, prefix=col_name)
169
+ dummies = dummies.reindex(columns=cols, fill_value=0)
170
+ feature_parts.append(dummies.values.astype(float))
171
+ else:
172
+ feature_parts.append(np.zeros((len(df), len(cols))))
173
+ except:
174
+ feature_parts.append(np.zeros((len(df), len(cols))))
175
+
176
+ # ===== CATEGORICAL: Label + Frequency =====
177
+ for col, le in self.encoders.items():
178
+ try:
179
+ if col in df.columns:
180
+ series = df[col].fillna('_MISSING_').astype(str).str.strip()
181
+ encoded = series.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
182
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
183
+ freq_key = f'{col}_freq_map'
184
+ if freq_key in self.transformers:
185
+ freq_map = self.transformers[freq_key]
186
+ freq_encoded = series.map(lambda s: freq_map.get(s, 0.0)).values.astype(float)
187
+ feature_parts.append(freq_encoded.reshape(-1, 1))
188
+ else:
189
+ feature_parts.append(np.zeros((len(df), 1)))
190
+ if f'{col}_freq_map' in self.transformers:
191
+ feature_parts.append(np.zeros((len(df), 1)))
192
+ except:
193
+ feature_parts.append(np.zeros((len(df), 1)))
194
+
195
+ # ===== CATEGORICAL INTERACTIONS =====
196
+ if 'cat_interactions' in self.transformers:
197
+ interaction_encoders = self.transformers['cat_interactions']
198
+ for key, value in interaction_encoders.items():
199
+ if key.endswith('_freq_map'):
200
+ continue
201
+ try:
202
+ col1, col2, le = value
203
+ if col1 in df.columns and col2 in df.columns:
204
+ combined = df[col1].fillna('_NA_').astype(str) + "_X_" + df[col2].fillna('_NA_').astype(str)
205
+ encoded = combined.map(lambda s: le.transform([s])[0] if s in le.classes_ else -1)
206
+ feature_parts.append(encoded.values.reshape(-1, 1).astype(float))
207
+ freq_key = f"{key}_freq_map"
208
+ if freq_key in interaction_encoders:
209
+ freq_map = interaction_encoders[freq_key]
210
+ freq_encoded = combined.map(lambda s: freq_map.get(s, 0.0)).values.astype(float)
211
+ feature_parts.append(freq_encoded.reshape(-1, 1))
212
+ else:
213
+ feature_parts.append(np.zeros((len(df), 1)))
214
+ if f"{key}_freq_map" in interaction_encoders:
215
+ feature_parts.append(np.zeros((len(df), 1)))
216
+ except:
217
+ pass
218
+
219
+ # ===== NUMERIC-CATEGORICAL AGGREGATES =====
220
+ if 'num_cat_aggs' in self.transformers:
221
+ agg_maps = self.transformers['num_cat_aggs']
222
+ for key, mean_map in agg_maps.items():
223
+ try:
224
+ parts = key.rsplit('_by_', 1)
225
+ if len(parts) == 2:
226
+ num_col = parts[0]
227
+ cat_col = parts[1].rsplit('_mean', 1)[0]
228
+ if num_col in df.columns and cat_col in df.columns:
229
+ cat_series = df[cat_col].fillna('_NA_').astype(str)
230
+ default_val = np.mean(list(mean_map.values())) if mean_map else 0
231
+ agg_values = cat_series.map(lambda s: mean_map.get(s, default_val)).values.astype(float)
232
+ feature_parts.append(agg_values.reshape(-1, 1))
233
+ deviation = df[num_col].fillna(0).values - agg_values
234
+ feature_parts.append(deviation.reshape(-1, 1))
235
+ else:
236
+ feature_parts.append(np.zeros((len(df), 1)))
237
+ feature_parts.append(np.zeros((len(df), 1)))
238
+ except:
239
+ pass
240
+
241
+ # ===== TEXT / NLP FEATURES =====
242
+ for key, tfidf in self.transformers.items():
243
+ if key.endswith('_tfidf'):
244
+ col = key.replace('_tfidf', '')
245
+ try:
246
+ if col in df.columns:
247
+ series = df[col].fillna('').astype(str)
248
+ cleaned = series.apply(self._clean_text)
249
+ tfidf_matrix = tfidf.transform(cleaned)
250
+ svd_key = f'{col}_svd'
251
+ if svd_key in self.transformers:
252
+ text_features = self.transformers[svd_key].transform(tfidf_matrix)
253
+ else:
254
+ text_features = tfidf_matrix.toarray()
255
+ feature_parts.append(text_features)
256
+ text_stats = np.array([self._text_statistics(t) for t in series])
257
+ feature_parts.append(text_stats)
258
+ sentiment = np.array([self._sentiment_score(t) for t in series])
259
+ feature_parts.append(sentiment)
260
+ except:
261
+ pass
262
+
263
+ # ===== COMBINE =====
264
+ if not feature_parts:
265
+ return np.zeros((len(df), len(self.feature_names)))
266
+
267
+ X = np.hstack(feature_parts)
268
+ X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
269
+
270
+ # Variance selector
271
+ if 'variance_selector' in self.transformers:
272
+ selector = self.transformers['variance_selector']
273
+ try:
274
+ if X.shape[1] == selector.n_features_in_:
275
+ X = selector.transform(X)
276
+ elif hasattr(self, 'selected_feature_indices') and self.selected_feature_indices is not None:
277
+ valid_idx = [i for i in self.selected_feature_indices if i < X.shape[1]]
278
+ if valid_idx:
279
+ X = X[:, valid_idx]
280
+ except:
281
+ pass
282
+
283
+ # PCA (if used during training)
284
+ if 'pca' in self.transformers:
285
+ try:
286
+ pca = self.transformers['pca']
287
+ if X.shape[1] == pca.n_features_in_:
288
+ X = pca.transform(X)
289
+ except:
290
+ pass
291
+
292
+ # Pad/truncate to match training dimensions
293
+ if X.shape[1] != len(self.feature_names):
294
+ if X.shape[1] < len(self.feature_names):
295
+ padding = np.zeros((len(df), len(self.feature_names) - X.shape[1]))
296
+ X = np.hstack([X, padding])
297
+ else:
298
+ X = X[:, :len(self.feature_names)]
299
+
300
+ return X
301
+
302
+ def transform_single(self, data):
303
+ for col in self.original_columns:
304
+ if col not in data:
305
+ data[col] = 0
306
+ df = pd.DataFrame([data])
307
+ X = self.transform(df)
308
+ expected = len(self.feature_names)
309
+ if X.shape[1] != expected:
310
+ if X.shape[1] < expected:
311
+ X = np.hstack([X, np.zeros((1, expected - X.shape[1]))])
312
+ else:
313
+ X = X[:, :expected]
314
+ return X
315
+
316
+
317
+ # =============================================================================
318
+ # Load model and feature engineer
319
+ # =============================================================================
320
+ FEATURE_ENGINEER = None
321
+
322
+ def _load_feature_engineer():
323
+ global FEATURE_ENGINEER
324
+ if FEATURE_ENGINEER is not None:
325
+ return FEATURE_ENGINEER
326
+ eng_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'feature_engineer.pkl')
327
+ if os.path.exists(eng_path):
328
+ try:
329
+ with open(eng_path, 'rb') as f:
330
+ eng_state = pickle.load(f)
331
+ FEATURE_ENGINEER = StandaloneFeatureEngineer(eng_state)
332
+ print(f" Pipeline: feature_engineer.pkl ({len(FEATURE_ENGINEER.feature_names)} engineered features)")
333
+ return FEATURE_ENGINEER
334
+ except Exception as e:
335
+ print(f" Warning: Could not load feature_engineer.pkl: {e}")
336
+ return None
337
+
338
+
339
+ def load_model(model_path='model.pkl'):
340
+ """Load the trained model and its components."""
341
+ if not os.path.exists(model_path):
342
+ print(f"❌ Model not found: {model_path}")
343
+ print(" Run the training script first, or ensure model.pkl is in this directory.")
344
+ return None
345
+
346
+ with open(model_path, 'rb') as f:
347
+ state = pickle.load(f)
348
+
349
+ print(f"✅ Loaded model: {state.get('model_name', 'Unknown')}")
350
+ print(f" Task: {state.get('task_type', 'Unknown')}")
351
+ print(f" Target: {state.get('target_column', 'Unknown')}")
352
+
353
+ # Load feature engineer (exact preprocessing pipeline)
354
+ _load_feature_engineer()
355
+ return state
356
+
357
+
358
+ def predict_single(state, input_data: dict):
359
+ """Make a prediction for a single data point.
360
+
361
+ Uses the EXACT same preprocessing pipeline as DataVision AutoML:
362
+ 1. StandaloneFeatureEngineer from feature_engineer.pkl (preferred)
363
+ 2. Fallback to basic label-encode + scale (less accurate)
364
+ """
365
+ model = state['model']
366
+ feature_cols = state.get('feature_columns', FEATURE_COLUMNS)
367
+ target_encoder = state.get('target_encoder', None)
368
+ task_type = state.get('task_type', 'classification')
369
+
370
+ # Use feature engineer for EXACT AutoML-matching preprocessing
371
+ engineer = _load_feature_engineer()
372
+ if engineer is not None:
373
+ try:
374
+ X = engineer.transform_single(input_data)
375
+ except Exception as e:
376
+ print(f"Warning: Feature engineer failed ({e}), using fallback")
377
+ X = _fallback_preprocess(input_data, state)
378
+ else:
379
+ X = _fallback_preprocess(input_data, state)
380
+
381
+ prediction = model.predict(X)[0]
382
+ result = {'raw_prediction': prediction}
383
+
384
+ if 'classif' in task_type.lower() and target_encoder is not None:
385
+ try:
386
+ result['prediction'] = target_encoder.inverse_transform([int(prediction)])[0]
387
+ except:
388
+ result['prediction'] = prediction
389
+ else:
390
+ result['prediction'] = float(prediction)
391
+
392
+ if 'classif' in task_type.lower() and hasattr(model, 'predict_proba'):
393
+ try:
394
+ proba = model.predict_proba(X)[0]
395
+ result['confidence'] = float(max(proba))
396
+ if target_encoder is not None:
397
+ result['probabilities'] = {
398
+ str(cls): float(p) for cls, p in zip(target_encoder.classes_, proba)
399
+ }
400
+ else:
401
+ result['probabilities'] = {str(i): float(p) for i, p in enumerate(proba)}
402
+ except:
403
+ pass
404
+
405
+ return result
406
+
407
+
408
+ def _fallback_preprocess(input_data, state):
409
+ """Fallback: basic label-encode + scale (used only if feature_engineer.pkl missing)."""
410
+ feature_cols = state.get('feature_columns', FEATURE_COLUMNS)
411
+ label_encoders = state.get('label_encoders', {})
412
+ scaler = state.get('scaler', None)
413
+
414
+ row = {}
415
+ for col in feature_cols:
416
+ val = input_data.get(col, 0)
417
+ if col in label_encoders:
418
+ le = label_encoders[col]
419
+ val_str = str(val)
420
+ val = le.transform([val_str])[0] if val_str in le.classes_ else 0
421
+ else:
422
+ try:
423
+ val = float(val)
424
+ except (ValueError, TypeError):
425
+ val = 0
426
+ row[col] = val
427
+
428
+ X_proc = pd.DataFrame([row], columns=feature_cols)
429
+ if scaler is not None:
430
+ try:
431
+ X_proc = pd.DataFrame(scaler.transform(X_proc), columns=feature_cols)
432
+ except:
433
+ pass
434
+ return X_proc
435
+
436
+
437
+ def predict_batch(state, csv_path: str, output_path: str = None):
438
+ """Make predictions for a CSV file."""
439
+ df = pd.read_csv(csv_path)
440
+ print(f"\n📊 Predicting for {len(df)} rows...")
441
+
442
+ predictions = []
443
+ for _, row in df.iterrows():
444
+ result = predict_single(state, row.to_dict())
445
+ predictions.append(result['prediction'])
446
+
447
+ df['prediction'] = predictions
448
+ if output_path is None:
449
+ output_path = csv_path.replace('.csv', '_predictions.csv')
450
+ df.to_csv(output_path, index=False)
451
+ print(f"💾 Predictions saved to: {output_path}")
452
+ return df
453
+
454
+
455
+ # =============================================================================
456
+ # Deep Learning Model Support (sklearn MLPClassifier/MLPRegressor)
457
+ # =============================================================================
458
+ DL_STATE = None
459
+
460
+ def load_dl_model(dl_path='deep_learning_model.pkl'):
461
+ """Load the Deep Learning model (sklearn MLP) with its preprocessing state."""
462
+ global DL_STATE
463
+ if not os.path.exists(dl_path):
464
+ return None
465
+ try:
466
+ with open(dl_path, 'rb') as f:
467
+ DL_STATE = pickle.load(f)
468
+ print(f"✅ Loaded DL model: {DL_STATE.get('algorithm', 'MLP')}")
469
+ print(f" Task: {DL_STATE.get('task_type', 'Unknown')}")
470
+ print(f" Features: {len(DL_STATE.get('feature_metadata', []))}")
471
+ return DL_STATE
472
+ except Exception as e:
473
+ print(f"⚠️ Failed to load DL model: {e}")
474
+ return None
475
+
476
+
477
+ def predict_single_dl(dl_state, input_data: dict):
478
+ """Make a prediction using the Deep Learning model.
479
+
480
+ Uses feature_metadata for correct preprocessing:
481
+ - Numeric features: float conversion, fillna with mean
482
+ - Categorical features: one-hot encoding matching training
483
+ - StandardScaler applied to full feature vector
484
+ """
485
+ model = dl_state['model']
486
+ scaler = dl_state.get('scaler')
487
+ label_encoder = dl_state.get('label_encoder')
488
+ feature_metadata = dl_state.get('feature_metadata', [])
489
+ task_type = dl_state.get('task_type', 'classification')
490
+ classes = dl_state.get('classes', [])
491
+
492
+ # Build feature vector using feature_metadata (same as DL engine)
493
+ feature_values = []
494
+
495
+ # Process numeric features first
496
+ for meta in feature_metadata:
497
+ if meta.get('type') == 'numeric':
498
+ col = meta['name']
499
+ if col in input_data:
500
+ try:
501
+ feature_values.append(float(input_data[col]))
502
+ except (ValueError, TypeError):
503
+ feature_values.append(meta.get('mean', 0))
504
+ else:
505
+ feature_values.append(meta.get('mean', 0))
506
+
507
+ # Process categorical features (one-hot encoded during training)
508
+ for meta in feature_metadata:
509
+ if meta.get('type') == 'categorical':
510
+ col = meta['name']
511
+ value = str(input_data.get(col, ''))
512
+ options = meta.get('options', [])
513
+
514
+ # One-hot encode: one value per option
515
+ for opt in options:
516
+ feature_values.append(1.0 if value == opt else 0.0)
517
+ # NaN dummy (dummy_na=True during training)
518
+ feature_values.append(1.0 if not value or value == 'nan' else 0.0)
519
+
520
+ # Fallback to feature_columns if no metadata
521
+ if not feature_values and dl_state.get('feature_columns'):
522
+ for col in dl_state['feature_columns']:
523
+ if col in input_data:
524
+ try:
525
+ feature_values.append(float(input_data[col]) if not isinstance(input_data[col], str) else 0)
526
+ except:
527
+ feature_values.append(0)
528
+ else:
529
+ feature_values.append(0)
530
+
531
+ X = np.array([feature_values])
532
+
533
+ # Handle dimension mismatch with scaler
534
+ if scaler and hasattr(scaler, 'n_features_in_'):
535
+ expected = scaler.n_features_in_
536
+ actual = X.shape[1]
537
+ if actual < expected:
538
+ padding = np.zeros((1, expected - actual))
539
+ X = np.hstack([X, padding])
540
+ elif actual > expected:
541
+ X = X[:, :expected]
542
+
543
+ # Scale
544
+ if scaler:
545
+ X_scaled = scaler.transform(X)
546
+ else:
547
+ X_scaled = X
548
+
549
+ # Predict
550
+ pred = model.predict(X_scaled)[0]
551
+ result = {'raw_prediction': pred}
552
+
553
+ if 'classif' in task_type.lower() and label_encoder is not None:
554
+ try:
555
+ result['prediction'] = str(label_encoder.inverse_transform([int(pred)])[0])
556
+ except:
557
+ result['prediction'] = str(pred)
558
+ else:
559
+ result['prediction'] = float(pred)
560
+
561
+ if 'classif' in task_type.lower() and hasattr(model, 'predict_proba'):
562
+ try:
563
+ proba = model.predict_proba(X_scaled)[0]
564
+ result['confidence'] = float(max(proba))
565
+ if classes:
566
+ result['probabilities'] = {
567
+ str(cls): round(float(p), 4) for cls, p in zip(classes, proba)
568
+ }
569
+ except:
570
+ pass
571
+
572
+ return result
573
+
574
+
575
+ def interactive_mode(state, use_dl=False):
576
+ """Interactive prediction mode with actual feature names and types."""
577
+ print("\n" + "=" * 50)
578
+ print("🔮 Interactive Prediction Mode")
579
+ if use_dl:
580
+ print(" (Using Deep Learning model)")
581
+ print("=" * 50)
582
+ print("Enter values for each feature:\n")
583
+
584
+ # For DL mode, use feature_metadata from DL state
585
+ if use_dl and DL_STATE:
586
+ input_data = {}
587
+ for meta in DL_STATE.get('feature_metadata', []):
588
+ col = meta['name']
589
+ ftype = meta.get('type', 'numeric')
590
+ if ftype == 'numeric':
591
+ mn = meta.get('min', '?')
592
+ mx = meta.get('max', '?')
593
+ default = meta.get('mean', 0)
594
+ val = input(f" {col} [{mn} - {mx}] (default: {default:.2f}): ").strip()
595
+ input_data[col] = float(val) if val else default
596
+ elif ftype == 'categorical':
597
+ options = meta.get('options', [])
598
+ print(f" {col} options: {options}")
599
+ val = input(f" {col} (pick one): ").strip()
600
+ input_data[col] = val if val else (options[0] if options else '')
601
+ result = predict_single_dl(DL_STATE, input_data)
602
+ else:
603
+ input_data = {}
604
+ for col, info in FEATURE_INFO.items():
605
+ ftype = info.get('type', 'numeric')
606
+
607
+ if ftype == 'numeric':
608
+ mn = info.get('min', '?')
609
+ mx = info.get('max', '?')
610
+ default = info.get('default', 0)
611
+ val = input(f" {col} [{mn} - {mx}] (default: {default}): ").strip()
612
+ input_data[col] = float(val) if val else default
613
+
614
+ elif ftype == 'categorical':
615
+ options = info.get('options', [])
616
+ print(f" {col} options: {options}")
617
+ val = input(f" {col} (pick one): ").strip()
618
+ input_data[col] = val if val else (options[0] if options else '')
619
+
620
+ elif ftype == 'text':
621
+ val = input(f" {col} (enter text): ").strip()
622
+ input_data[col] = val if val else ''
623
+ result = predict_single(state, input_data)
624
+
625
+ print(f"\n🎯 Prediction: {result['prediction']}")
626
+ if 'confidence' in result:
627
+ print(f"📊 Confidence: {result['confidence']:.1%}")
628
+ if 'probabilities' in result:
629
+ print(f"📊 Probabilities:")
630
+ for cls, prob in sorted(result['probabilities'].items(), key=lambda x: x[1], reverse=True):
631
+ bar = "█" * int(prob * 30)
632
+ print(f" {cls:>15}: {prob:.1%} {bar}")
633
+
634
+ return result
635
+
636
+
637
+ def main():
638
+ parser = argparse.ArgumentParser(description="Make predictions with your trained model")
639
+ parser.add_argument('--model', type=str, default='model.pkl', help='Path to model file')
640
+ parser.add_argument('--dl-model', type=str, default='deep_learning_model.pkl', help='Path to DL model file')
641
+ parser.add_argument('--mode', type=str, default='auto', choices=['auto', 'traditional', 'dl'], help='Prediction mode')
642
+ parser.add_argument('--json', type=str, default=None, help='JSON string with input data')
643
+ parser.add_argument('--csv', type=str, default=None, help='CSV file for batch predictions')
644
+ parser.add_argument('--output', type=str, default=None, help='Output CSV for batch predictions')
645
+ parser.add_argument('--interactive', action='store_true', help='Interactive mode')
646
+ args = parser.parse_args()
647
+
648
+ # Auto-detect which model to use based on BEST_MODE from training
649
+ use_dl = False
650
+ dl_state = None
651
+ state = None
652
+
653
+ # Use DL if: explicitly requested, or BEST_MODE is deep_learning, or DL model exists
654
+ if args.mode == 'dl' or (args.mode == 'auto' and BEST_MODE == 'deep_learning') or (args.mode == 'auto' and os.path.exists(args.dl_model) and not os.path.exists(args.model)):
655
+ dl_state = load_dl_model(args.dl_model)
656
+ if dl_state:
657
+ use_dl = True
658
+
659
+ if not use_dl:
660
+ state = load_model(args.model)
661
+ if state is None:
662
+ # Last resort: try DL model
663
+ dl_state = load_dl_model(args.dl_model)
664
+ if dl_state:
665
+ use_dl = True
666
+ else:
667
+ return
668
+
669
+ if args.json:
670
+ input_data = json.loads(args.json)
671
+ if use_dl:
672
+ result = predict_single_dl(dl_state, input_data)
673
+ else:
674
+ result = predict_single(state, input_data)
675
+ print(f"\n🎯 Prediction: {result['prediction']}")
676
+ if 'confidence' in result:
677
+ print(f"📊 Confidence: {result['confidence']:.1%}")
678
+ print(f"\n📋 Full result: {json.dumps(result, indent=2, default=str)}")
679
+ elif args.csv:
680
+ if use_dl:
681
+ # Batch DL predictions
682
+ df = pd.read_csv(args.csv)
683
+ print(f"\n📊 Predicting for {len(df)} rows (DL mode)...")
684
+ predictions = []
685
+ for _, row in df.iterrows():
686
+ result = predict_single_dl(dl_state, row.to_dict())
687
+ predictions.append(result['prediction'])
688
+ df['prediction'] = predictions
689
+ output_path = args.output or args.csv.replace('.csv', '_predictions.csv')
690
+ df.to_csv(output_path, index=False)
691
+ print(f"💾 Predictions saved to: {output_path}")
692
+ else:
693
+ predict_batch(state, args.csv, args.output)
694
+ else:
695
+ interactive_mode(state, use_dl=use_dl)
696
+
697
+
698
+ # =========================================================================
699
+ # Example (using YOUR actual feature names):
700
+ #
701
+ # from predict import load_model, predict_single
702
+ # state = load_model('model.pkl')
703
+ # result = predict_single(state, {
704
+ # 'battery_power': 1238.51, # numeric, range [510.0 .. 1987.0]
705
+ # 'blue': 0.49, # numeric, range [0.0 .. 1.0]
706
+ # 'clock_speed': 1.52, # numeric, range [0.5 .. 3.0]
707
+ # 'dual_sim': 0.51, # numeric, range [0.0 .. 1.0]
708
+ # 'fc': 4.29, # numeric, range [0.0 .. 16.0]
709
+ # 'four_g': 0.52, # numeric, range [0.0 .. 1.0]
710
+ # 'int_memory': 32.05, # numeric, range [2.0 .. 64.0]
711
+ # 'm_dep': 0.50, # numeric, range [0.1 .. 1.0]
712
+ # 'mobile_wt': 140.24, # numeric, range [80.0 .. 199.0]
713
+ # 'n_cores': 4.52, # numeric, range [1.0 .. 8.0]
714
+ # 'pc': 9.92, # numeric, range [0.0 .. 20.0]
715
+ # 'px_height': 644.46, # numeric, range [15.0 .. 1791.01]
716
+ # 'ram': 2124.26, # numeric, range [296.99 .. 3958.01]
717
+ # 'sc_h': 12.31, # numeric, range [5.0 .. 19.0]
718
+ # 'sc_w': 5.76, # numeric, range [0.0 .. 17.0]
719
+ # 'three_g': 1.00, # numeric, range [1.0 .. 1.0]
720
+ # 'touch_screen': 0.50, # numeric, range [0.0 .. 1.0]
721
+ # 'wifi': 0.51, # numeric, range [0.0 .. 1.0]
722
+ # })
723
+ # print(result['prediction'])
724
+ # =========================================================================
725
+
726
+
727
+ if __name__ == '__main__':
728
+ main()
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/requirements.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ flask>=2.2.0
2
+ imbalanced-learn>=0.10.0
3
+ joblib>=1.1.0
4
+ matplotlib>=3.5.0
5
+ numpy>=1.21.0
6
+ pandas>=1.3.0
7
+ scikit-learn>=1.0.0
8
+ scipy>=1.7.0
9
+ seaborn>=0.12.0
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/train_traditional.py ADDED
@@ -0,0 +1,894 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🚀 Traditional ML Training Script — YOUR DataVision AutoML Results
3
+ =====================================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ ╔═══════════════════════════════════════════════════════════════════════╗
7
+ ║ 📦 model.pkl contains your ALREADY TRAINED model with the metrics ║
8
+ ║ you saw in the AutoML UI. ║
9
+ ║ ║
10
+ ║ ▶ python evaluate.py — See your EXACT AutoML results ║
11
+ ║ ▶ python predict.py — Make predictions with your model ║
12
+ ║ ║
13
+ ║ This script is for RETRAINING only. Results may differ slightly ║
14
+ ║ due to train/test random split, even with same hyperparameters. ║
15
+ ╚═══════════════════════════════════════════════════════════════════════╝
16
+
17
+ YOUR AUTOML TRAINING RESULTS:
18
+ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
19
+ 🏆 WINNING MODEL : RandomForest
20
+ 🎯 Target : price_range
21
+ 📊 Task : classification
22
+ 📁 Features : 20 columns
23
+
24
+ YOUR METRICS:
25
+ # accuracy: 0.8925
26
+ # f1: 0.8929
27
+ # precision: 0.8937
28
+ # recall: 0.8925
29
+ # roc_auc: 0.9796
30
+
31
+ YOUR LEADERBOARD (what AutoML trained):
32
+ # 1. ?: 0.8925
33
+
34
+ FEATURE INFO:
35
+ # battery_power: numeric (min=510.0, max=1987.0, mean=1238.513)
36
+ # blue: numeric (min=0.0, max=1.0, mean=0.495)
37
+ # clock_speed: numeric (min=0.5, max=3.0, mean=1.52225)
38
+ # dual_sim: numeric (min=0.0, max=1.0, mean=0.5095)
39
+ # fc: numeric (min=0.0, max=16.0, mean=4.294)
40
+ # four_g: numeric (min=0.0, max=1.0, mean=0.5215)
41
+ # int_memory: numeric (min=2.0, max=64.0, mean=32.0465)
42
+ # m_dep: numeric (min=0.1, max=1.0, mean=0.50175)
43
+ # mobile_wt: numeric (min=80.0, max=199.0, mean=140.2395)
44
+ # n_cores: numeric (min=1.0, max=8.0, mean=4.5205)
45
+
46
+ Usage:
47
+ # VERIFY YOUR AUTOML RESULTS:
48
+ python evaluate.py # Shows exact AutoML metrics
49
+
50
+ # MAKE PREDICTIONS:
51
+ python predict.py --json '{"col1": value}' # Your trained model
52
+
53
+ # RETRAIN (if needed):
54
+ python train_traditional.py # Train ONLY RandomForest
55
+ python train_traditional.py --all # Train ALL algorithms
56
+ """
57
+
58
+ import os
59
+ import pickle
60
+ import argparse
61
+ import time
62
+ import warnings
63
+ import numpy as np
64
+ import pandas as pd
65
+ from scipy import stats as scipy_stats
66
+ from sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score
67
+ from sklearn.preprocessing import LabelEncoder, RobustScaler, StandardScaler
68
+ from sklearn.feature_selection import VarianceThreshold
69
+ from sklearn.impute import KNNImputer
70
+ from sklearn.metrics import (
71
+ accuracy_score, precision_score, recall_score, f1_score,
72
+ r2_score, mean_absolute_error, mean_squared_error,
73
+ classification_report, confusion_matrix
74
+ )
75
+ from sklearn.ensemble import (
76
+ RandomForestClassifier, RandomForestRegressor,
77
+ GradientBoostingClassifier, GradientBoostingRegressor,
78
+ ExtraTreesClassifier, ExtraTreesRegressor,
79
+ HistGradientBoostingClassifier, HistGradientBoostingRegressor,
80
+ AdaBoostClassifier, AdaBoostRegressor,
81
+ BaggingClassifier, BaggingRegressor,
82
+ )
83
+ from sklearn.linear_model import (
84
+ LogisticRegression, Ridge, Lasso, ElasticNet,
85
+ SGDClassifier, SGDRegressor, RidgeClassifier,
86
+ PassiveAggressiveClassifier, PassiveAggressiveRegressor,
87
+ BayesianRidge
88
+ )
89
+ from sklearn.svm import SVC, SVR
90
+ from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
91
+ from sklearn.naive_bayes import GaussianNB, BernoulliNB
92
+ from sklearn.neural_network import MLPClassifier, MLPRegressor
93
+ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
94
+
95
+ warnings.filterwarnings('ignore')
96
+
97
+ HAS_XGB = False
98
+ HAS_LGB = False
99
+ HAS_CB = False
100
+ HAS_SMOTE = False
101
+
102
+ try:
103
+ import xgboost as xgb
104
+ HAS_XGB = True
105
+ except ImportError:
106
+ pass
107
+ try:
108
+ import lightgbm as lgb
109
+ HAS_LGB = True
110
+ except ImportError:
111
+ pass
112
+ try:
113
+ from catboost import CatBoostClassifier, CatBoostRegressor
114
+ HAS_CB = True
115
+ except ImportError:
116
+ pass
117
+ try:
118
+ from imblearn.over_sampling import SMOTE
119
+ HAS_SMOTE = True
120
+ except ImportError:
121
+ pass
122
+
123
+ TARGET_COLUMN = 'price_range'
124
+ TASK_TYPE = 'classification'
125
+
126
+ # YOUR WINNING MODEL FROM AUTOML
127
+ BEST_MODEL_NAME = 'RandomForest'
128
+
129
+
130
+ # ============================================================================
131
+ # STEP 1: DATA CLEANING (mirrors ProductionDataCleaner.clean())
132
+ # ============================================================================
133
+
134
+ def clean_data(df):
135
+ """
136
+ Mirrors DataVision's ProductionDataCleaner pipeline:
137
+ 1. Remove useless columns (IDs, URLs, high-cardinality strings)
138
+ 2. Remove exact duplicates
139
+ 3. Strip whitespace, replace 'nan'/'None'/'' with NaN
140
+ 4. Smart type conversion (parse $, M/K units, commas)
141
+ 5. KNN imputation for numerics (median fallback)
142
+ 6. Mode fill for categoricals
143
+ 7. Replace inf/-inf with median
144
+ 8. Remove constant columns (nunique <= 1)
145
+ 9. IQR-based outlier capping (Q1-3*IQR to Q3+3*IQR, clipped to [1st, 99th] percentile)
146
+ 10. Final NaN fill to 0
147
+ """
148
+ print("\n🧹 Step 1: Data Cleaning (ProductionDataCleaner)")
149
+ print("-" * 50)
150
+ original_shape = df.shape
151
+
152
+ # Remove useless columns
153
+ id_patterns = ['unnamed', 'index', '_id', 'guid', 'uuid', 'url', 'link', 'href', 'path']
154
+ cols_to_drop = []
155
+ for col in df.columns:
156
+ col_lower = col.lower().strip()
157
+ if any(pat in col_lower for pat in id_patterns):
158
+ if col != TARGET_COLUMN:
159
+ cols_to_drop.append(col)
160
+ elif df[col].dtype == 'object' and col != TARGET_COLUMN:
161
+ unique_ratio = df[col].nunique() / max(len(df), 1)
162
+ if unique_ratio > 0.9:
163
+ cols_to_drop.append(col)
164
+ if cols_to_drop:
165
+ df = df.drop(columns=cols_to_drop, errors='ignore')
166
+ print(f" Dropped {len(cols_to_drop)} useless columns: {cols_to_drop[:5]}")
167
+
168
+ # Remove exact duplicates
169
+ before = len(df)
170
+ df = df.drop_duplicates()
171
+ if len(df) < before:
172
+ print(f" Removed {before - len(df)} duplicate rows")
173
+
174
+ # Strip whitespace + replace string NaN variants
175
+ for col in df.select_dtypes(include=['object']).columns:
176
+ df[col] = df[col].str.strip()
177
+ df[col] = df[col].replace(
178
+ ['nan', 'NaN', 'None', 'null', 'NULL', 'none', 'NA', 'N/A', ''],
179
+ np.nan
180
+ )
181
+
182
+ # Smart type conversion (parse $, M/K, commas)
183
+ for col in df.select_dtypes(include=['object']).columns:
184
+ if col == TARGET_COLUMN:
185
+ continue
186
+ sample = df[col].dropna().head(20)
187
+ if len(sample) == 0:
188
+ continue
189
+ converted = 0
190
+ for val in sample:
191
+ try:
192
+ v = str(val).replace('$', '').replace(',', '').strip()
193
+ if v.upper().endswith('M'):
194
+ float(v[:-1])
195
+ elif v.upper().endswith('K'):
196
+ float(v[:-1])
197
+ elif v.endswith('+'):
198
+ float(v[:-1])
199
+ else:
200
+ float(v)
201
+ converted += 1
202
+ except:
203
+ pass
204
+ if converted / max(len(sample), 1) > 0.6:
205
+ def parse_value(val):
206
+ if pd.isna(val):
207
+ return np.nan
208
+ v = str(val).replace('$', '').replace(',', '').strip()
209
+ try:
210
+ if v.upper().endswith('M'):
211
+ return float(v[:-1]) * 1_000_000
212
+ elif v.upper().endswith('K'):
213
+ return float(v[:-1]) * 1_000
214
+ elif v.endswith('+'):
215
+ return float(v[:-1])
216
+ return float(v)
217
+ except:
218
+ return np.nan
219
+ df[col] = df[col].apply(parse_value)
220
+ print(f" Converted {col} to numeric (parsed special formats)")
221
+
222
+ # KNN imputation for numeric columns
223
+ numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
224
+ if TARGET_COLUMN in numeric_cols:
225
+ numeric_cols.remove(TARGET_COLUMN)
226
+ if numeric_cols and df[numeric_cols].isna().any().any():
227
+ try:
228
+ imputer = KNNImputer(n_neighbors=5, weights='distance')
229
+ df[numeric_cols] = imputer.fit_transform(df[numeric_cols])
230
+ print(f" KNN imputed {len(numeric_cols)} numeric columns")
231
+ except:
232
+ for col in numeric_cols:
233
+ df[col] = df[col].fillna(df[col].median())
234
+ print(f" Median-filled {len(numeric_cols)} numeric columns (KNN fallback)")
235
+
236
+ # Mode fill for categoricals
237
+ for col in df.select_dtypes(include=['object', 'category']).columns:
238
+ if col == TARGET_COLUMN:
239
+ continue
240
+ mode = df[col].mode()
241
+ df[col] = df[col].fillna(mode.iloc[0] if len(mode) > 0 else '_MISSING_')
242
+
243
+ # Replace inf/-inf with median
244
+ for col in numeric_cols:
245
+ if np.isinf(df[col]).any():
246
+ med = df[col].replace([np.inf, -np.inf], np.nan).median()
247
+ df[col] = df[col].replace([np.inf, -np.inf], med)
248
+
249
+ # Remove constant columns
250
+ const_cols = [col for col in df.columns if col != TARGET_COLUMN and df[col].nunique() <= 1]
251
+ if const_cols:
252
+ df = df.drop(columns=const_cols)
253
+ print(f" Removed {len(const_cols)} constant columns")
254
+
255
+ # IQR-based outlier capping (same as DataVision)
256
+ capped_count = 0
257
+ for col in df.select_dtypes(include=[np.number]).columns:
258
+ if col == TARGET_COLUMN:
259
+ continue
260
+ Q1 = df[col].quantile(0.25)
261
+ Q3 = df[col].quantile(0.75)
262
+ IQR = Q3 - Q1
263
+ p1 = df[col].quantile(0.01)
264
+ p99 = df[col].quantile(0.99)
265
+ lower = max(p1, Q1 - 3 * IQR)
266
+ upper = min(p99, Q3 + 3 * IQR)
267
+ before_clip = ((df[col] < lower) | (df[col] > upper)).sum()
268
+ if before_clip > 0:
269
+ df[col] = df[col].clip(lower=lower, upper=upper)
270
+ capped_count += before_clip
271
+ if capped_count:
272
+ print(f" Capped {capped_count} outlier values (IQR method)")
273
+
274
+ # Final NaN fill
275
+ for col in df.select_dtypes(include=[np.number]).columns:
276
+ df[col] = df[col].fillna(0)
277
+
278
+ print(f" Clean: {original_shape} → {df.shape}")
279
+ return df
280
+
281
+
282
+ # ============================================================================
283
+ # STEP 2: FEATURE ENGINEERING (mirrors ProductionFeatureEngineer)
284
+ # ============================================================================
285
+
286
+ def engineer_features(X, mode='fast'):
287
+ """
288
+ Mirrors DataVision's ProductionFeatureEngineer.fit_transform():
289
+ 1. Detect column types (numeric, categorical, text, datetime)
290
+ 2. Log-transform skewed numerics (skewness > 2)
291
+ 3. RobustScaler for numeric features
292
+ 4. Polynomial features (squared + interactions for top columns)
293
+ 5. Categorical: binary→LabelEncoder, low-card→one-hot, high-card→label+frequency
294
+ 6. VarianceThreshold (remove near-constant features, threshold=0.01)
295
+ """
296
+ print("\n🔧 Step 2: Feature Engineering (ProductionFeatureEngineer)")
297
+ print("-" * 50)
298
+
299
+ numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
300
+ categorical_cols = X.select_dtypes(include=['object', 'category']).columns.tolist()
301
+
302
+ label_encoders = {}
303
+
304
+ # --- Numeric feature engineering ---
305
+ # Log transform skewed features (skewness > 2, non-negative only)
306
+ log_transformed = []
307
+ for col in numeric_cols:
308
+ if (X[col] >= 0).all():
309
+ skew = X[col].skew()
310
+ if abs(skew) > 2:
311
+ X[f'{col}_log'] = np.log1p(X[col])
312
+ log_transformed.append(col)
313
+ if log_transformed:
314
+ print(f" Log-transformed {len(log_transformed)} skewed features")
315
+
316
+ # Scale with RobustScaler (same as DataVision)
317
+ numeric_cols_updated = X.select_dtypes(include=[np.number]).columns.tolist()
318
+ scaler = RobustScaler()
319
+ if numeric_cols_updated:
320
+ X[numeric_cols_updated] = scaler.fit_transform(X[numeric_cols_updated])
321
+
322
+ # Polynomial features (top 4 numeric columns — squared + interactions)
323
+ if 2 <= len(numeric_cols) <= 10:
324
+ top_n = min(4, len(numeric_cols))
325
+ top_cols = numeric_cols[:top_n]
326
+ for col in top_cols:
327
+ X[f'{col}_sq'] = X[col] ** 2
328
+ for i in range(len(top_cols)):
329
+ for j in range(i + 1, len(top_cols)):
330
+ X[f'{top_cols[i]}_x_{top_cols[j]}'] = X[top_cols[i]] * X[top_cols[j]]
331
+ print(f" Added polynomial features for top {top_n} numeric columns")
332
+
333
+ # --- Categorical feature engineering ---
334
+ for col in categorical_cols:
335
+ nuniq = X[col].nunique()
336
+ if nuniq <= 2:
337
+ # Binary: LabelEncoder
338
+ le = LabelEncoder()
339
+ X[col] = le.fit_transform(X[col].astype(str))
340
+ label_encoders[col] = le
341
+ elif nuniq <= 10:
342
+ # Low cardinality: one-hot encoding (no drop_first — same as DataVision)
343
+ dummies = pd.get_dummies(X[col], prefix=col, drop_first=False)
344
+ X = pd.concat([X.drop(columns=[col]), dummies], axis=1)
345
+ else:
346
+ # High cardinality: LabelEncoder + frequency encoding
347
+ le = LabelEncoder()
348
+ X[f'{col}_label'] = le.fit_transform(X[col].astype(str))
349
+ freq = X[col].value_counts(normalize=True)
350
+ X[f'{col}_freq'] = X[col].map(freq)
351
+ label_encoders[col] = le
352
+ X = X.drop(columns=[col])
353
+
354
+ # Force all to float
355
+ for col in X.columns:
356
+ try:
357
+ X[col] = X[col].astype(float)
358
+ except:
359
+ le = LabelEncoder()
360
+ X[col] = le.fit_transform(X[col].astype(str)).astype(float)
361
+
362
+ # Handle NaN/Inf
363
+ X = X.replace([np.inf, -np.inf], 0)
364
+ X = X.fillna(0)
365
+
366
+ # VarianceThreshold (remove near-constant features)
367
+ try:
368
+ vt = VarianceThreshold(threshold=0.01)
369
+ cols_before = X.columns.tolist()
370
+ X_filtered = pd.DataFrame(vt.fit_transform(X), columns=X.columns[vt.get_support()], index=X.index)
371
+ removed = len(cols_before) - X_filtered.shape[1]
372
+ if removed > 0:
373
+ print(f" VarianceThreshold removed {removed} near-constant features")
374
+ X = X_filtered
375
+ except:
376
+ pass
377
+
378
+ print(f" Final features: {X.shape[1]}")
379
+ return X, label_encoders, scaler
380
+
381
+
382
+ # ============================================================================
383
+ # STEP 3: YOUR EXACT TRAINED MODEL
384
+ # ============================================================================
385
+
386
+ # These are the EXACT hyperparameters from YOUR trained model
387
+ ACTUAL_MODEL_CLASS = 'RandomForestClassifier'
388
+ ACTUAL_MODEL_PARAMS = {'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': 'balanced', 'criterion': 'gini', 'max_depth': 10, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'monotonic_cst': None, 'n_estimators': 150, 'random_state': 42}
389
+
390
+ def get_exact_trained_model():
391
+ """Returns YOUR exact model with the SAME hyperparameters as AutoML used.
392
+
393
+ This ensures retraining produces the SAME results as your AutoML session.
394
+ """
395
+ params = ACTUAL_MODEL_PARAMS.copy()
396
+
397
+ # Remove non-constructor / problematic params that cause crashes
398
+ for bad_key in ['oob_score', 'warm_start', 'verbose', 'callbacks',
399
+ 'early_stopping_rounds', 'use_label_encoder', 'gpu_id',
400
+ 'predictor', 'enable_categorical', 'feature_types',
401
+ 'validate_parameters', 'nthread', 'device',
402
+ 'n_jobs', 'silent', 'importance_type',
403
+ 'cat_features', 'text_features', 'task_type',
404
+ 'multi_strategy', 'max_cat_to_onehot', 'max_cat_threshold',
405
+ 'num_parallel_tree', 'monotone_constraints',
406
+ 'interaction_constraints', 'grow_policy', 'max_leaves',
407
+ 'process_type', 'updater', 'refresh_leaf', 'max_bin',
408
+ 'bootstrap_type', 'devices', 'embedding_features']:
409
+ params.pop(bad_key, None)
410
+ # Remove private params
411
+ params = {k: v for k, v in params.items() if not k.startswith('_')}
412
+
413
+
414
+ return {'RandomForest': RandomForestClassifier(**params)}
415
+
416
+
417
+ def get_single_model(model_name, n_samples):
418
+ """Get ONLY your winning model.
419
+
420
+ Uses get_exact_trained_model() to get YOUR exact hyperparameters.
421
+ """
422
+ # First try to use exact trained model params
423
+ if model_name == BEST_MODEL_NAME or model_name == 'RandomForest':
424
+ return get_exact_trained_model()
425
+
426
+ # Otherwise fall back to generic models
427
+ is_large = n_samples > 50000
428
+ all_models = get_models(n_samples, mode='fast')
429
+
430
+ # Try exact match first
431
+ if model_name in all_models:
432
+ return {model_name: all_models[model_name]}
433
+
434
+ # Try case-insensitive match
435
+ model_name_lower = model_name.lower().replace('_', '').replace(' ', '')
436
+ for k, v in all_models.items():
437
+ if k.lower().replace('_', '').replace(' ', '') == model_name_lower:
438
+ return {k: v}
439
+
440
+ # Check in ultra models too
441
+ all_models = get_models(n_samples, mode='ultra')
442
+ if model_name in all_models:
443
+ return {model_name: all_models[model_name]}
444
+ for k, v in all_models.items():
445
+ if k.lower().replace('_', '').replace(' ', '') == model_name_lower:
446
+ return {k: v}
447
+
448
+ # Fallback to exact trained model
449
+ print(f" ⚠️ Model '{model_name}' not found, using your trained model")
450
+ return get_exact_trained_model()
451
+
452
+
453
+ def get_models(n_samples, mode='fast'):
454
+ """Returns the EXACT algorithms & hyperparameters used by DataVision AutoML.
455
+
456
+ Mirrors ProductionModelTrainer.get_models() with the same conditional
457
+ logic for dataset size and mode.
458
+ """
459
+ is_large = n_samples > 50000
460
+ is_ultra = mode == 'ultra'
461
+
462
+
463
+ models = {
464
+ 'LogisticRegression': LogisticRegression(
465
+ max_iter=2000, C=0.1, random_state=42, n_jobs=1,
466
+ class_weight='balanced'
467
+ ),
468
+ 'RandomForest': RandomForestClassifier(
469
+ n_estimators=100 if is_large else (150 if not is_ultra else 250),
470
+ max_depth=8 if is_large else (12 if not is_ultra else 18),
471
+ min_samples_split=4, random_state=42, n_jobs=-1,
472
+ class_weight='balanced'
473
+ ),
474
+ 'GradientBoosting': GradientBoostingClassifier(
475
+ n_estimators=80 if is_large else 120,
476
+ max_depth=5 if is_large else 6,
477
+ learning_rate=0.1, random_state=42, subsample=0.8
478
+ ),
479
+ 'HistGradientBoosting': HistGradientBoostingClassifier(
480
+ max_iter=100 if is_large else (150 if not is_ultra else 250),
481
+ max_depth=6 if is_large else (8 if not is_ultra else 12),
482
+ learning_rate=0.1, random_state=42, class_weight='balanced'
483
+ ),
484
+ 'GaussianNB': GaussianNB(),
485
+ 'ExtraTrees': ExtraTreesClassifier(
486
+ n_estimators=80 if is_large else (100 if not is_ultra else 200),
487
+ max_depth=8 if is_large else (10 if not is_ultra else 15),
488
+ random_state=42, n_jobs=-1, class_weight='balanced'
489
+ ),
490
+ }
491
+
492
+ if not is_large:
493
+ models['KNN'] = KNeighborsClassifier(n_neighbors=7, weights='distance', n_jobs=-1)
494
+
495
+ if HAS_XGB:
496
+ models['XGBoost'] = xgb.XGBClassifier(
497
+ n_estimators=100 if is_large else (150 if not is_ultra else 300),
498
+ max_depth=6 if is_large else (7 if not is_ultra else 10),
499
+ learning_rate=0.1 if not is_ultra else 0.05,
500
+ subsample=0.8, colsample_bytree=0.8,
501
+ random_state=42, n_jobs=-1, verbosity=0,
502
+ eval_metric='mlogloss',
503
+ tree_method='hist' if is_large else 'auto',
504
+ scale_pos_weight=10
505
+ )
506
+ if HAS_LGB:
507
+ models['LightGBM'] = lgb.LGBMClassifier(
508
+ n_estimators=100 if is_large else (150 if not is_ultra else 300),
509
+ max_depth=6 if is_large else (7 if not is_ultra else 10),
510
+ learning_rate=0.1 if not is_ultra else 0.05,
511
+ num_leaves=31 if not is_ultra else 50,
512
+ subsample=0.8, colsample_bytree=0.8,
513
+ random_state=42, n_jobs=-1, verbose=-1,
514
+ class_weight='balanced'
515
+ )
516
+ if HAS_CB:
517
+ if is_ultra:
518
+ models['CatBoost'] = CatBoostClassifier(
519
+ iterations=150 if is_large else 300, depth=6 if is_large else 10,
520
+ learning_rate=0.05 if is_large else 0.03,
521
+ l2_leaf_reg=3, random_strength=1, bagging_temperature=0.5,
522
+ random_state=42, verbose=False,
523
+ auto_class_weights='Balanced', allow_writing_files=False
524
+ )
525
+ else:
526
+ models['CatBoost'] = CatBoostClassifier(
527
+ iterations=100, depth=6, learning_rate=0.1,
528
+ random_state=42, verbose=False,
529
+ auto_class_weights='Balanced', allow_writing_files=False
530
+ )
531
+
532
+ # Ultra mode adds more algorithms
533
+ if is_ultra:
534
+ if not is_large:
535
+ models['SVM_RBF'] = SVC(kernel='rbf', C=1.0, gamma='scale', probability=True, random_state=42, class_weight='balanced')
536
+ models['SVM_Linear'] = SVC(kernel='linear', C=0.5, probability=True, random_state=42, class_weight='balanced')
537
+ models['RidgeClassifier'] = RidgeClassifier(alpha=1.0, random_state=42, class_weight='balanced')
538
+ models['MLP'] = MLPClassifier(
539
+ hidden_layer_sizes=(128, 64) if is_large else (256, 128, 64, 32),
540
+ max_iter=500 if is_large else 1500,
541
+ learning_rate='adaptive', early_stopping=True,
542
+ batch_size=256 if is_large else 'auto',
543
+ validation_fraction=0.1, random_state=42
544
+ )
545
+ models['AdaBoost'] = AdaBoostClassifier(n_estimators=100 if is_large else 150, learning_rate=0.5, random_state=42)
546
+ models['Bagging'] = BaggingClassifier(
547
+ n_estimators=50 if is_large else 100,
548
+ max_samples=0.5 if is_large else 0.8,
549
+ max_features=0.8, random_state=42, n_jobs=-1
550
+ )
551
+ models['SGD'] = SGDClassifier(loss='log_loss', max_iter=1500, random_state=42, class_weight='balanced', early_stopping=True)
552
+ models['PassiveAggressive'] = PassiveAggressiveClassifier(max_iter=1500, random_state=42, class_weight='balanced')
553
+ models['DecisionTree'] = DecisionTreeClassifier(max_depth=12 if is_large else 18, min_samples_split=3, random_state=42, class_weight='balanced')
554
+ models['BernoulliNB'] = BernoulliNB(alpha=0.5)
555
+
556
+ return models
557
+
558
+
559
+ # ============================================================================
560
+ # STEP 4: TRAINING (mirrors ProductionModelTrainer.train_all())
561
+ # ============================================================================
562
+
563
+ def apply_smote(X_train, y_train):
564
+ """Apply SMOTE for imbalanced classification (same as DataVision).
565
+
566
+ Only applied when:
567
+ - imbalance_ratio > 1.5
568
+ - min class count >= 6
569
+ Falls back to class_weight='balanced' if SMOTE fails.
570
+ """
571
+ if not HAS_SMOTE or TASK_TYPE != 'classification':
572
+ return X_train, y_train, False
573
+
574
+ class_counts = pd.Series(y_train).value_counts()
575
+ max_count = class_counts.max()
576
+ min_count = class_counts.min()
577
+ imbalance_ratio = max_count / max(min_count, 1)
578
+
579
+ if imbalance_ratio <= 1.5 or min_count < 6:
580
+ return X_train, y_train, False
581
+
582
+ try:
583
+ k_neighbors = min(5, min_count - 1)
584
+ n_classes = len(class_counts)
585
+
586
+ if n_classes == 2:
587
+ if imbalance_ratio > 100:
588
+ sampling_strategy = 0.1
589
+ elif imbalance_ratio > 20:
590
+ sampling_strategy = 0.25
591
+ else:
592
+ sampling_strategy = 1.0
593
+ else:
594
+ sampling_strategy = 'not majority' if imbalance_ratio > 20 else 'auto'
595
+
596
+ smote = SMOTE(k_neighbors=k_neighbors, sampling_strategy=sampling_strategy, random_state=42)
597
+ X_resampled, y_resampled = smote.fit_resample(X_train, y_train)
598
+ print(f" ⚖️ SMOTE applied: {len(X_train)} → {len(X_resampled)} samples (ratio: {imbalance_ratio:.1f}x)")
599
+ return X_resampled, y_resampled, True
600
+ except Exception as e:
601
+ print(f" ⚠️ SMOTE failed ({e}), using class_weight='balanced'")
602
+ return X_train, y_train, False
603
+
604
+
605
+ def train_and_evaluate(X_train, X_test, y_train, y_test, mode='fast', selected_models=None):
606
+ """Train models with real DataVision scoring logic.
607
+
608
+ Args:
609
+ selected_models: Optional dict of {name: model} to train only specific models.
610
+ If None, trains all models for the specified mode.
611
+
612
+ Scoring:
613
+ - Classification (balanced): F1-macro
614
+ - Classification (imbalanced >5x): minority_recall*0.6 + f1_macro*0.4
615
+ - Regression: R² score
616
+
617
+ Cross-validation:
618
+ - Fast: 3-fold StratifiedKFold/KFold
619
+ - Ultra: 5-fold
620
+ """
621
+ is_clf = TASK_TYPE == 'classification'
622
+ is_ultra = mode == 'ultra'
623
+
624
+ # Apply SMOTE (classification only)
625
+ X_train_sm, y_train_sm, smote_applied = apply_smote(X_train, y_train)
626
+
627
+ # Use selected_models if provided, otherwise get all models for the mode
628
+ if selected_models is not None:
629
+ models = selected_models
630
+ else:
631
+ models = get_models(len(X_train), mode)
632
+ results = []
633
+ best_model = None
634
+ best_score = -999
635
+ best_name = ""
636
+
637
+ # Check imbalance for scoring
638
+ if is_clf:
639
+ class_counts = pd.Series(y_train).value_counts()
640
+ imbalance_ratio = class_counts.max() / max(class_counts.min(), 1)
641
+ is_imbalanced = imbalance_ratio > 5
642
+ minority_class = class_counts.idxmin()
643
+
644
+ # Cross-validation setup (same as DataVision)
645
+ cv_folds = 3 if mode == 'fast' else 5
646
+ if is_clf:
647
+ cv = StratifiedKFold(n_splits=cv_folds, shuffle=True, random_state=42)
648
+ else:
649
+ cv = KFold(n_splits=cv_folds, shuffle=True, random_state=42)
650
+
651
+ print(f"\n🏋️ Training {len(models)} model(s) ({'ALL' if selected_models is None else 'SELECTED'} mode, {cv_folds}-fold CV)...")
652
+ print("=" * 70)
653
+
654
+ for name, model in models.items():
655
+ try:
656
+ t0 = time.time()
657
+ model.fit(X_train_sm, y_train_sm)
658
+ y_pred = model.predict(X_test)
659
+ elapsed = time.time() - t0
660
+
661
+ if is_clf:
662
+ f1_macro = f1_score(y_test, y_pred, average='macro', zero_division=0)
663
+
664
+ if is_imbalanced and len(pd.Series(y_train).unique()) == 2:
665
+ # DataVision's real scoring for imbalanced binary:
666
+ # minority_recall * 0.6 + f1_macro * 0.4
667
+ from sklearn.metrics import recall_score as rs
668
+ minority_recall = rs(y_test, y_pred, pos_label=minority_class, zero_division=0)
669
+ score = minority_recall * 0.6 + f1_macro * 0.4
670
+ else:
671
+ score = f1_macro
672
+
673
+ metrics = {
674
+ 'accuracy': accuracy_score(y_test, y_pred),
675
+ 'precision': precision_score(y_test, y_pred, average='weighted', zero_division=0),
676
+ 'recall': recall_score(y_test, y_pred, average='weighted', zero_division=0),
677
+ 'f1_weighted': f1_score(y_test, y_pred, average='weighted', zero_division=0),
678
+ 'f1_macro': f1_macro,
679
+ 'score': score,
680
+ }
681
+ else:
682
+ score = r2_score(y_test, y_pred)
683
+ metrics = {
684
+ 'r2': score,
685
+ 'mae': mean_absolute_error(y_test, y_pred),
686
+ 'rmse': np.sqrt(mean_squared_error(y_test, y_pred)),
687
+ 'score': score,
688
+ }
689
+
690
+ # Cross-validation
691
+ try:
692
+ cv_scoring = 'f1_weighted' if is_clf else 'r2'
693
+ cv_scores = cross_val_score(model, X_train_sm, y_train_sm, cv=cv, scoring=cv_scoring)
694
+ metrics['cv_mean'] = cv_scores.mean()
695
+ metrics['cv_std'] = cv_scores.std()
696
+ except:
697
+ metrics['cv_mean'] = score
698
+ metrics['cv_std'] = 0
699
+
700
+ results.append({
701
+ 'name': name, 'score': score, 'metrics': metrics, 'model': model,
702
+ 'y_pred': y_pred, 'time': elapsed, 'cv_mean': metrics.get('cv_mean', score)
703
+ })
704
+
705
+ marker = "🏆" if score > best_score else " "
706
+ cv_str = f"CV: {metrics.get('cv_mean', 0):.4f}±{metrics.get('cv_std', 0):.3f}"
707
+ print(f" {marker} {name:.<30} Score: {score:.4f} | {cv_str} ({elapsed:.1f}s)")
708
+
709
+ if score > best_score:
710
+ best_score = score
711
+ best_model = model
712
+ best_name = name
713
+
714
+ except Exception as e:
715
+ print(f" ❌ {name:.<30} Failed: {str(e)[:60]}")
716
+
717
+ # Sort results by score
718
+ results.sort(key=lambda x: x['score'], reverse=True)
719
+
720
+ print("=" * 70)
721
+
722
+ if not results:
723
+ print("\n❌ All models failed to train!")
724
+ print(" Try: python train_traditional.py --all (to try all algorithms)")
725
+ return None, '', {}, []
726
+
727
+ print(f"\n🏆 Best Model: {best_name} (Score: {best_score:.4f})")
728
+
729
+ best_result = results[0]
730
+ if is_clf:
731
+ print(f"\n📊 Classification Report:")
732
+ print(classification_report(y_test, best_result['y_pred']))
733
+ else:
734
+ print(f"\n📊 Regression Metrics:")
735
+ for k, v in best_result['metrics'].items():
736
+ if k not in ('score', 'cv_mean', 'cv_std'):
737
+ print(f" {k.upper()}: {v:.4f}")
738
+
739
+ # Overfitting check (same as DataVision's Production Intelligence)
740
+ y_pred_train = best_model.predict(X_train)
741
+ if is_clf:
742
+ train_score = accuracy_score(y_train, y_pred_train)
743
+ test_score = accuracy_score(y_test, best_result['y_pred'])
744
+ else:
745
+ train_score = r2_score(y_train, y_pred_train)
746
+ test_score = r2_score(y_test, best_result['y_pred'])
747
+ gap = train_score - test_score
748
+ if gap > 0.1:
749
+ print(f"\n⚠️ Overfitting detected (train-test gap: {gap:.4f})")
750
+ elif gap < 0.02:
751
+ print(f"\n✅ Good generalization (train-test gap: {gap:.4f})")
752
+
753
+ return best_model, best_name, best_result['metrics'], results
754
+
755
+
756
+ def main():
757
+ parser = argparse.ArgumentParser(description="Train ML model (reproduces your DataVision AutoML)")
758
+ parser.add_argument('--data', type=str, default='data/cleaned_data.csv', help='Path to CSV data file')
759
+ parser.add_argument('--output', type=str, default='model.pkl', help='Output model file')
760
+ parser.add_argument('--test-size', type=float, default=0.2, help='Test split ratio (DataVision uses 0.2)')
761
+ parser.add_argument('--all', action='store_true', help='Train ALL algorithms (like full AutoML)')
762
+ parser.add_argument('--mode', type=str, default='fast', choices=['fast', 'ultra'],
763
+ help='Training mode when using --all (fast: ~10 models, ultra: ~25 models)')
764
+ args = parser.parse_args()
765
+
766
+ print("\n" + "=" * 70)
767
+ if args.all:
768
+ print("🚀 DataVision AutoML — Training ALL Algorithms")
769
+ else:
770
+ print(f"🚀 DataVision AutoML — Training YOUR Winning Model: RandomForest")
771
+ print(" (Use --all to train all algorithms like full AutoML)")
772
+ print("=" * 70)
773
+
774
+ if not os.path.exists(args.data):
775
+ print(f"❌ Data file not found: {args.data}")
776
+ return
777
+
778
+ start_time = time.time()
779
+
780
+ # Load data
781
+ df = pd.read_csv(args.data)
782
+
783
+ # Step 1: Clean data (ProductionDataCleaner)
784
+ df = clean_data(df)
785
+ df = df.dropna(subset=[TARGET_COLUMN])
786
+
787
+ # Separate target BEFORE feature engineering (anti-leakage — same as DataVision)
788
+ X = df.drop(columns=[TARGET_COLUMN])
789
+ y = df[TARGET_COLUMN]
790
+
791
+ # Encode target
792
+ target_encoder = None
793
+ if TASK_TYPE == 'classification':
794
+ target_encoder = LabelEncoder()
795
+ y = pd.Series(target_encoder.fit_transform(y), name=TARGET_COLUMN)
796
+ print(f"\n🎯 Target: price_range ({len(target_encoder.classes_)} classes: {list(target_encoder.classes_)[:10]})")
797
+ else:
798
+ print(f"\n🎯 Target: price_range (regression)")
799
+
800
+ # Train/test split BEFORE feature engineering (anti-leakage)
801
+ X_train, X_test, y_train, y_test = train_test_split(
802
+ X, y, test_size=args.test_size, random_state=42,
803
+ stratify=y if TASK_TYPE == 'classification' else None
804
+ )
805
+ print(f"📦 Split: {len(X_train)} train / {len(X_test)} test")
806
+ print(f" ⚠️ Feature engineering applied on train ONLY (anti-leakage)")
807
+
808
+ # Step 2: Feature engineering (fit on train, transform test — anti-leakage)
809
+ X_train_eng, label_encoders, scaler = engineer_features(X_train.copy(), mode=args.mode)
810
+
811
+ # Transform test set using same encoding (simplified — uses same column structure)
812
+ X_test_eng = X_test.copy()
813
+ # Apply same encoding to test
814
+ for col in X_test_eng.select_dtypes(include=['object', 'category']).columns:
815
+ if col in label_encoders:
816
+ le = label_encoders[col]
817
+ X_test_eng[col] = X_test_eng[col].astype(str).map(
818
+ lambda x: le.transform([x])[0] if x in le.classes_ else 0
819
+ )
820
+ else:
821
+ le = LabelEncoder()
822
+ X_test_eng[col] = le.fit_transform(X_test_eng[col].astype(str))
823
+ for col in X_test_eng.select_dtypes(include=['object', 'category']).columns:
824
+ X_test_eng[col] = pd.to_numeric(X_test_eng[col], errors='coerce').fillna(0)
825
+ X_test_eng = X_test_eng.fillna(0)
826
+ X_test_eng = X_test_eng.replace([np.inf, -np.inf], 0)
827
+
828
+ # Ensure same columns as train
829
+ for col in X_train_eng.columns:
830
+ if col not in X_test_eng.columns:
831
+ X_test_eng[col] = 0
832
+ X_test_eng = X_test_eng[X_train_eng.columns]
833
+ for col in X_test_eng.columns:
834
+ X_test_eng[col] = pd.to_numeric(X_test_eng[col], errors='coerce').fillna(0)
835
+
836
+ # Step 3 & 4: Train models
837
+ # DEFAULT: Train ONLY your winning model (to reproduce exact result)
838
+ # --all: Train all algorithms like full AutoML
839
+ if args.all:
840
+ print(f"\n📊 Training ALL models (mode={args.mode})...")
841
+ best_model, best_name, metrics, all_results = train_and_evaluate(
842
+ X_train_eng, X_test_eng, y_train, y_test, mode=args.mode
843
+ )
844
+ else:
845
+ print(f"\n📊 Training ONLY your winning model: RandomForest")
846
+ models = get_single_model(BEST_MODEL_NAME, len(X_train_eng))
847
+ best_model, best_name, metrics, all_results = train_and_evaluate(
848
+ X_train_eng, X_test_eng, y_train, y_test, mode=args.mode, selected_models=models
849
+ )
850
+
851
+ if best_model is None:
852
+ print("\n⚠️ Selected model failed. Retrying with ALL algorithms...")
853
+ best_model, best_name, metrics, all_results = train_and_evaluate(
854
+ X_train_eng, X_test_eng, y_train, y_test, mode='fast'
855
+ )
856
+
857
+ if best_model is None:
858
+ print("\n❌ All models failed. Check your data and dependencies.")
859
+ return
860
+
861
+ total_time = time.time() - start_time
862
+
863
+ # Save model state (same structure as DataVision's engine_state)
864
+ engine_state = {
865
+ 'model': best_model,
866
+ 'model_name': best_name,
867
+ 'task_type': TASK_TYPE,
868
+ 'task_type_simple': TASK_TYPE,
869
+ 'target_column': TARGET_COLUMN,
870
+ 'feature_columns': X_train_eng.columns.tolist(),
871
+ 'label_encoders': label_encoders,
872
+ 'target_encoder': target_encoder,
873
+ 'scaler': scaler,
874
+ 'metrics': metrics,
875
+ 'production_mode': True,
876
+ 'mode': args.mode,
877
+ 'training_time_seconds': total_time,
878
+ 'leaderboard': [
879
+ {'name': r['name'], 'score': r['score'], 'metrics': r['metrics'], 'time': r.get('time', 0)}
880
+ for r in all_results
881
+ ],
882
+ }
883
+
884
+ with open(args.output, 'wb') as f:
885
+ pickle.dump(engine_state, f)
886
+
887
+ file_size = os.path.getsize(args.output) / 1024
888
+ print(f"\n💾 Model saved to: {args.output} ({file_size:.1f} KB)")
889
+ print(f"⏱️ Total time: {total_time:.1f}s")
890
+ print("✅ Training complete!")
891
+
892
+
893
+ if __name__ == '__main__':
894
+ main()
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/utils/__init__.py ADDED
File without changes
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/utils/preprocessing.py ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🔧 Preprocessing Utilities — Mirrors DataVision AutoML Pipeline
3
+ ===================================================================
4
+ These utilities replicate the EXACT preprocessing pipeline used by
5
+ DataVision's ProductionDataCleaner + ProductionFeatureEngineer.
6
+
7
+ Pipeline steps:
8
+ 1. Remove useless columns (IDs, URLs, high-cardinality strings)
9
+ 2. Remove duplicates
10
+ 3. Smart type conversion (parse $, M/K units, commas)
11
+ 4. KNN imputation for numerics (median fallback)
12
+ 5. Mode fill for categoricals
13
+ 6. Replace inf/-inf with median
14
+ 7. Remove constant columns
15
+ 8. IQR-based outlier capping
16
+ 9. Log-transform skewed features
17
+ 10. RobustScaler for numeric features
18
+ 11. Categorical encoding (binary→label, low-card→one-hot, high-card→label+freq)
19
+ 12. VarianceThreshold to remove near-constant features
20
+ """
21
+
22
+ import numpy as np
23
+ import pandas as pd
24
+ from sklearn.preprocessing import LabelEncoder, RobustScaler
25
+ from sklearn.feature_selection import VarianceThreshold
26
+ from sklearn.impute import KNNImputer
27
+
28
+ TARGET_COLUMN = 'price_range'
29
+ TASK_TYPE = 'classification'
30
+ FEATURE_COLUMNS = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt', 'n_cores', 'pc', 'px_height', 'px_width', 'ram', 'sc_h', 'sc_w', 'talk_time', 'three_g', 'touch_screen', 'wifi']
31
+
32
+
33
+ def detect_task_type(y):
34
+ """Auto-detect classification vs regression."""
35
+ if y.dtype == 'object' or y.nunique() < 20:
36
+ return 'classification'
37
+ return 'regression'
38
+
39
+
40
+ def remove_useless_columns(df, target_col=TARGET_COLUMN):
41
+ """Remove ID-like, URL, and high-cardinality columns (same as DataVision)."""
42
+ id_patterns = ['unnamed', 'index', '_id', 'guid', 'uuid', 'url', 'link', 'href', 'path']
43
+ cols_to_drop = []
44
+ for col in df.columns:
45
+ if col == target_col:
46
+ continue
47
+ col_lower = col.lower().strip()
48
+ if any(pat in col_lower for pat in id_patterns):
49
+ cols_to_drop.append(col)
50
+ elif df[col].dtype == 'object' and df[col].nunique() / max(len(df), 1) > 0.9:
51
+ cols_to_drop.append(col)
52
+ return df.drop(columns=cols_to_drop, errors='ignore') if cols_to_drop else df
53
+
54
+
55
+ def smart_type_conversion(df, target_col=TARGET_COLUMN):
56
+ """Parse hidden numerics: $, M/K units, commas (same as DataVision)."""
57
+ for col in df.select_dtypes(include=['object']).columns:
58
+ if col == target_col:
59
+ continue
60
+ sample = df[col].dropna().head(20)
61
+ if len(sample) == 0:
62
+ continue
63
+ converted = 0
64
+ for val in sample:
65
+ try:
66
+ v = str(val).replace('$', '').replace(',', '').strip()
67
+ if v.upper().endswith(('M', 'K', '+')):
68
+ float(v[:-1])
69
+ else:
70
+ float(v)
71
+ converted += 1
72
+ except:
73
+ pass
74
+ if converted / max(len(sample), 1) > 0.6:
75
+ def parse_val(val):
76
+ if pd.isna(val):
77
+ return np.nan
78
+ v = str(val).replace('$', '').replace(',', '').strip()
79
+ try:
80
+ if v.upper().endswith('M'): return float(v[:-1]) * 1e6
81
+ if v.upper().endswith('K'): return float(v[:-1]) * 1e3
82
+ if v.endswith('+'): return float(v[:-1])
83
+ return float(v)
84
+ except:
85
+ return np.nan
86
+ df[col] = df[col].apply(parse_val)
87
+ return df
88
+
89
+
90
+ def impute_missing(df, target_col=TARGET_COLUMN):
91
+ """KNN imputation for numerics, mode fill for categoricals (same as DataVision)."""
92
+ numeric_cols = [c for c in df.select_dtypes(include=[np.number]).columns if c != target_col]
93
+ if numeric_cols and df[numeric_cols].isna().any().any():
94
+ try:
95
+ imputer = KNNImputer(n_neighbors=5, weights='distance')
96
+ df[numeric_cols] = imputer.fit_transform(df[numeric_cols])
97
+ except:
98
+ for col in numeric_cols:
99
+ df[col] = df[col].fillna(df[col].median())
100
+
101
+ for col in df.select_dtypes(include=['object', 'category']).columns:
102
+ if col == target_col:
103
+ continue
104
+ mode = df[col].mode()
105
+ df[col] = df[col].fillna(mode.iloc[0] if len(mode) > 0 else '_MISSING_')
106
+ return df
107
+
108
+
109
+ def cap_outliers_iqr(df, target_col=TARGET_COLUMN):
110
+ """IQR-based outlier capping with [1st, 99th] percentile bounds (same as DataVision)."""
111
+ for col in df.select_dtypes(include=[np.number]).columns:
112
+ if col == target_col:
113
+ continue
114
+ Q1, Q3 = df[col].quantile(0.25), df[col].quantile(0.75)
115
+ IQR = Q3 - Q1
116
+ p1, p99 = df[col].quantile(0.01), df[col].quantile(0.99)
117
+ lower = max(p1, Q1 - 3 * IQR)
118
+ upper = min(p99, Q3 + 3 * IQR)
119
+ df[col] = df[col].clip(lower=lower, upper=upper)
120
+ return df
121
+
122
+
123
+ def engineer_features(X):
124
+ """Feature engineering matching DataVision's ProductionFeatureEngineer."""
125
+ label_encoders = {}
126
+ numeric_cols = X.select_dtypes(include=[np.number]).columns.tolist()
127
+ categorical_cols = X.select_dtypes(include=['object', 'category']).columns.tolist()
128
+
129
+ # Log-transform skewed numerics (skewness > 2, non-negative)
130
+ for col in numeric_cols:
131
+ if (X[col] >= 0).all() and abs(X[col].skew()) > 2:
132
+ X[f'{col}_log'] = np.log1p(X[col])
133
+
134
+ # RobustScaler (same as DataVision, not StandardScaler)
135
+ numeric_cols_all = X.select_dtypes(include=[np.number]).columns.tolist()
136
+ scaler = RobustScaler()
137
+ if numeric_cols_all:
138
+ X[numeric_cols_all] = scaler.fit_transform(X[numeric_cols_all])
139
+
140
+ # Polynomial features (top 4 numeric, squared + interactions)
141
+ if 2 <= len(numeric_cols) <= 10:
142
+ top_n = min(4, len(numeric_cols))
143
+ top_cols = numeric_cols[:top_n]
144
+ for col in top_cols:
145
+ X[f'{col}_sq'] = X[col] ** 2
146
+ for i in range(len(top_cols)):
147
+ for j in range(i + 1, len(top_cols)):
148
+ X[f'{top_cols[i]}_x_{top_cols[j]}'] = X[top_cols[i]] * X[top_cols[j]]
149
+
150
+ # Categorical encoding
151
+ for col in categorical_cols:
152
+ nuniq = X[col].nunique()
153
+ if nuniq <= 2: # Binary → LabelEncoder
154
+ le = LabelEncoder()
155
+ X[col] = le.fit_transform(X[col].astype(str))
156
+ label_encoders[col] = le
157
+ elif nuniq <= 10: # Low cardinality → one-hot
158
+ dummies = pd.get_dummies(X[col], prefix=col, drop_first=False)
159
+ X = pd.concat([X.drop(columns=[col]), dummies], axis=1)
160
+ else: # High cardinality → label + frequency encoding
161
+ le = LabelEncoder()
162
+ X[f'{col}_label'] = le.fit_transform(X[col].astype(str))
163
+ freq = X[col].value_counts(normalize=True)
164
+ X[f'{col}_freq'] = X[col].map(freq)
165
+ label_encoders[col] = le
166
+ X = X.drop(columns=[col])
167
+
168
+ # Force float + clean
169
+ for col in X.columns:
170
+ try:
171
+ X[col] = X[col].astype(float)
172
+ except:
173
+ le = LabelEncoder()
174
+ X[col] = le.fit_transform(X[col].astype(str)).astype(float)
175
+ X = X.replace([np.inf, -np.inf], 0).fillna(0)
176
+
177
+ # VarianceThreshold (remove near-constant, threshold=0.01)
178
+ try:
179
+ vt = VarianceThreshold(threshold=0.01)
180
+ X = pd.DataFrame(vt.fit_transform(X), columns=X.columns[vt.get_support()], index=X.index)
181
+ except:
182
+ pass
183
+
184
+ return X, label_encoders, scaler
185
+
186
+
187
+ def full_preprocess(df, target_column=TARGET_COLUMN):
188
+ """Complete DataVision preprocessing pipeline.
189
+
190
+ Combines ProductionDataCleaner.clean() + ProductionFeatureEngineer.fit_transform().
191
+ """
192
+ # Step 1: Clean
193
+ df = df.dropna(subset=[target_column])
194
+ df = df.drop_duplicates()
195
+
196
+ # Strip whitespace and replace string NaN variants
197
+ for col in df.select_dtypes(include=['object']).columns:
198
+ df[col] = df[col].str.strip()
199
+ df[col] = df[col].replace(['nan', 'NaN', 'None', 'null', 'NULL', ''], np.nan)
200
+
201
+ df = remove_useless_columns(df, target_column)
202
+ df = smart_type_conversion(df, target_column)
203
+ df = impute_missing(df, target_column)
204
+
205
+ # Replace inf, remove constant columns
206
+ for col in df.select_dtypes(include=[np.number]).columns:
207
+ if col != target_column:
208
+ df[col] = df[col].replace([np.inf, -np.inf], df[col].replace([np.inf, -np.inf], np.nan).median())
209
+ const_cols = [c for c in df.columns if c != target_column and df[c].nunique() <= 1]
210
+ df = df.drop(columns=const_cols, errors='ignore')
211
+
212
+ df = cap_outliers_iqr(df, target_column)
213
+ df[df.select_dtypes(include=[np.number]).columns] = df.select_dtypes(include=[np.number]).fillna(0)
214
+
215
+ # Step 2: Split target
216
+ X = df.drop(columns=[target_column])
217
+ y = df[target_column]
218
+
219
+ task_type = detect_task_type(y)
220
+ target_encoder = None
221
+ if task_type == 'classification':
222
+ target_encoder = LabelEncoder()
223
+ y = pd.Series(target_encoder.fit_transform(y))
224
+
225
+ # Step 3: Engineer features
226
+ X, label_encoders, scaler = engineer_features(X)
227
+ feature_cols = X.columns.tolist()
228
+
229
+ return X, y, task_type, target_encoder, label_encoders, scaler, feature_cols
backend/.workspace/982669a5-01d4-48a7-b23d-670b0e18cd72/visualize.py ADDED
@@ -0,0 +1,944 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 📊 Visualization Script — ALL Charts Matching DataVision AutoML
3
+ ================================================================
4
+ Auto-generated by DataVision AI
5
+
6
+ Model : RandomForest
7
+ Target Column : price_range
8
+ Task Type : classification
9
+ Features : 20 columns
10
+
11
+ Supports Traditional ML AND Deep Learning models.
12
+ Automatically loads the correct model based on BEST_MODE.
13
+
14
+ Generates the SAME charts you see in DataVision AutoML:
15
+ Classification: confusion matrix, class distribution, ROC curve,
16
+ precision-recall, calibration, confidence histogram,
17
+ metrics heatmap, feature importance, model comparison,
18
+ correlation heatmap, feature distributions, boxplots
19
+ Regression: actual vs predicted, residuals analysis, error distribution,
20
+ prediction overview, feature importance, model comparison,
21
+ correlation heatmap, feature distributions, boxplots
22
+ Deep Learning: training loss curve, validation score curve,
23
+ model architecture summary, MLP weight importance
24
+ Unsupervised: KMeans clustering scatter plot (PCA 2D),
25
+ cluster distribution chart
26
+
27
+ Usage:
28
+ python visualize.py # Auto-detect best model
29
+ python visualize.py --mode dl # Force Deep Learning model
30
+ python visualize.py --mode traditional # Force traditional ML model
31
+ python visualize.py --data my_data.csv
32
+ python visualize.py --output charts/
33
+ """
34
+
35
+ import os
36
+ import pickle
37
+ import argparse
38
+ import warnings
39
+ import numpy as np
40
+ import pandas as pd
41
+ import matplotlib
42
+ matplotlib.use('Agg')
43
+ import matplotlib.pyplot as plt
44
+ import seaborn as sns
45
+ from sklearn.model_selection import train_test_split
46
+ from sklearn.preprocessing import LabelEncoder, label_binarize
47
+ from sklearn.metrics import (
48
+ accuracy_score, confusion_matrix, classification_report,
49
+ r2_score, mean_absolute_error, mean_squared_error,
50
+ roc_curve, auc, precision_recall_curve, average_precision_score,
51
+ precision_score, recall_score, f1_score
52
+ )
53
+
54
+ try:
55
+ from sklearn.calibration import calibration_curve
56
+ HAS_CALIBRATION = True
57
+ except ImportError:
58
+ HAS_CALIBRATION = False
59
+
60
+ warnings.filterwarnings('ignore')
61
+ sns.set_style("whitegrid")
62
+ plt.rcParams.update({
63
+ 'figure.figsize': (10, 6),
64
+ 'font.size': 12,
65
+ 'axes.titlesize': 14,
66
+ 'axes.labelsize': 12,
67
+ })
68
+
69
+ TARGET_COLUMN = 'price_range'
70
+ TASK_TYPE = 'classification'
71
+ BEST_MODE = 'traditional' # Which mode won during AutoML
72
+ NUMERIC_FEATURES = ['battery_power', 'blue', 'clock_speed', 'dual_sim', 'fc', 'four_g', 'int_memory', 'm_dep', 'mobile_wt']
73
+ LEADERBOARD_NAMES = ['?']
74
+ LEADERBOARD_SCORES = [0.8925]
75
+
76
+
77
+ def load_model_and_data(model_path='model.pkl', data_path='data/cleaned_data.csv'):
78
+ if not os.path.exists(model_path):
79
+ print(f"❌ Model not found: {model_path}")
80
+ return None, None
81
+ with open(model_path, 'rb') as f:
82
+ state = pickle.load(f)
83
+ df = pd.read_csv(data_path) if os.path.exists(data_path) else None
84
+ return state, df
85
+
86
+
87
+ def load_dl_model(dl_path='deep_learning_model.pkl'):
88
+ """Load the Deep Learning model (sklearn MLP)."""
89
+ if not os.path.exists(dl_path):
90
+ return None
91
+ try:
92
+ with open(dl_path, 'rb') as f:
93
+ state = pickle.load(f)
94
+ print(f"✅ Loaded DL model: {state.get('algorithm', 'MLP')}")
95
+ print(f" Task: {state.get('task_type', 'Unknown')}")
96
+ print(f" Features: {len(state.get('feature_columns', state.get('feature_metadata', [])))}")
97
+ return state
98
+ except Exception as e:
99
+ print(f"⚠️ Failed to load DL model: {e}")
100
+ return None
101
+
102
+
103
+ def generate_charts(state, df, output_dir='charts'):
104
+ os.makedirs(output_dir, exist_ok=True)
105
+
106
+ model = state['model']
107
+ task_type = state.get('task_type', 'classification')
108
+ target_col = state.get('target_column', TARGET_COLUMN)
109
+ feature_cols = state.get('feature_columns', [])
110
+ is_clf = 'classif' in task_type.lower()
111
+ is_dl = state.get('model_type') == 'deep_learning'
112
+ charts = []
113
+
114
+ # Load SAVED predictions from AutoML (exact results, no feature mismatch)
115
+ y_test = np.array(state['y_test']) if state.get('y_test') is not None else None
116
+ y_pred = np.array(state['y_pred']) if state.get('y_pred') is not None else None
117
+ y_proba = np.array(state['y_proba']) if state.get('y_proba') is not None else None
118
+ # Handle both ML (target_encoder) and DL (label_encoder) naming
119
+ target_encoder = state.get('target_encoder', None)
120
+ label_encoder = state.get('label_encoder', None)
121
+ encoder = target_encoder or label_encoder
122
+ class_names = list(encoder.classes_) if encoder is not None else None
123
+ mode_display = 'Deep Learning (MLP)' if is_dl else 'Traditional ML'
124
+ model_name = state.get('model_name', state.get('algorithm', 'Unknown'))
125
+
126
+ print(f"\n📊 Generating charts for {model_name} ({mode_display})...")
127
+ print(f" Saved predictions: y_test={y_test is not None}, y_pred={y_pred is not None}, y_proba={y_proba is not None}")
128
+
129
+ # =====================================================================
130
+ # 1. MODEL COMPARISON / LEADERBOARD (same as AutoML)
131
+ # =====================================================================
132
+ if LEADERBOARD_NAMES and LEADERBOARD_SCORES:
133
+ try:
134
+ fig, ax = plt.subplots(figsize=(10, max(4, len(LEADERBOARD_NAMES) * 0.5)))
135
+ colors = sns.color_palette('viridis', len(LEADERBOARD_NAMES))
136
+ y_pos = range(len(LEADERBOARD_NAMES))
137
+ ax.barh(y_pos, LEADERBOARD_SCORES, color=colors)
138
+ ax.set_yticks(y_pos)
139
+ ax.set_yticklabels(LEADERBOARD_NAMES)
140
+ metric_label = 'F1-Score' if is_clf else 'R²'
141
+ ax.set_xlabel(metric_label)
142
+ ax.set_title(f'Model Leaderboard — {metric_label}')
143
+ for i, v in enumerate(LEADERBOARD_SCORES):
144
+ ax.text(v + 0.001, i, f'{v:.4f}', va='center', fontsize=9)
145
+ plt.tight_layout()
146
+ path = os.path.join(output_dir, 'model_comparison.png')
147
+ plt.savefig(path, dpi=150, bbox_inches='tight')
148
+ plt.close()
149
+ charts.append(path)
150
+ print(f" ✅ Model Comparison → {path}")
151
+ except Exception as e:
152
+ print(f" ⚠️ Model Comparison failed: {e}")
153
+
154
+ # =====================================================================
155
+ # 2. FEATURE IMPORTANCE (from model)
156
+ # =====================================================================
157
+ try:
158
+ importances = None
159
+ if hasattr(model, 'feature_importances_'):
160
+ # Tree-based models (RF, XGB, etc.)
161
+ importances = model.feature_importances_
162
+ elif hasattr(model, 'coefs_') and len(model.coefs_) > 0:
163
+ # MLP/Neural Network — use mean absolute weight of first layer
164
+ importances = np.mean(np.abs(model.coefs_[0]), axis=1)
165
+ elif hasattr(model, 'coef_'):
166
+ # Linear models (LogReg, SVM, etc.)
167
+ coef = model.coef_
168
+ importances = np.mean(np.abs(coef), axis=0) if coef.ndim > 1 else np.abs(coef)
169
+
170
+ if importances is not None:
171
+ names = feature_cols[:len(importances)] if feature_cols else [f'Feature_{i}' for i in range(len(importances))]
172
+ fi = pd.Series(importances, index=names).nlargest(min(20, len(names)))
173
+
174
+ fig, ax = plt.subplots(figsize=(10, max(4, len(fi) * 0.35)))
175
+ bars = fi.sort_values().plot(kind='barh', ax=ax,
176
+ color=plt.cm.viridis(np.linspace(0.2, 0.8, len(fi))))
177
+ title_suffix = ' (MLP Weight Magnitude)' if is_dl else ''
178
+ ax.set_title(f'Feature Importance — Top Features{title_suffix}')
179
+ ax.set_xlabel('Importance')
180
+ for i, (val, name) in enumerate(zip(fi.sort_values(), fi.sort_values().index)):
181
+ ax.text(val + 0.001, i, f'{val:.4f}', va='center', fontsize=8)
182
+ plt.tight_layout()
183
+ path = os.path.join(output_dir, 'feature_importance.png')
184
+ plt.savefig(path, dpi=150, bbox_inches='tight')
185
+ plt.close()
186
+ charts.append(path)
187
+ print(f" ✅ Feature Importance → {path}")
188
+ except Exception as e:
189
+ print(f" ⚠️ Feature Importance failed: {e}")
190
+
191
+ # =====================================================================
192
+ # DL-SPECIFIC: TRAINING HISTORY (loss curve + validation score)
193
+ # =====================================================================
194
+ if is_dl:
195
+ training_history = state.get('training_history', {})
196
+ loss_curve = training_history.get('loss', [])
197
+ val_scores = training_history.get('val_score', [])
198
+
199
+ if loss_curve:
200
+ try:
201
+ n_plots = 1 + (1 if val_scores else 0)
202
+ fig, axes = plt.subplots(1, n_plots, figsize=(6 * n_plots, 5))
203
+ if n_plots == 1:
204
+ axes = [axes]
205
+
206
+ # Loss curve
207
+ axes[0].plot(range(1, len(loss_curve) + 1), loss_curve, color='#EF4444', lw=2)
208
+ axes[0].set_xlabel('Epoch')
209
+ axes[0].set_ylabel('Loss')
210
+ axes[0].set_title('Training Loss Curve')
211
+ axes[0].grid(True, alpha=0.3)
212
+
213
+ # Validation score curve
214
+ if val_scores and n_plots > 1:
215
+ axes[1].plot(range(1, len(val_scores) + 1), val_scores, color='#22C55E', lw=2)
216
+ axes[1].set_xlabel('Epoch')
217
+ axes[1].set_ylabel('Validation Score')
218
+ axes[1].set_title('Validation Score Curve')
219
+ axes[1].grid(True, alpha=0.3)
220
+
221
+ plt.suptitle(f'Deep Learning Training History — {state.get("algorithm", "MLP")}', fontsize=14)
222
+ plt.tight_layout()
223
+ path = os.path.join(output_dir, 'training_history.png')
224
+ plt.savefig(path, dpi=150, bbox_inches='tight')
225
+ plt.close()
226
+ charts.append(path)
227
+ print(f" ✅ Training History → {path}")
228
+ except Exception as e:
229
+ print(f" ⚠️ Training History failed: {e}")
230
+
231
+ # DL Model Architecture Summary
232
+ try:
233
+ fig, ax = plt.subplots(figsize=(8, 4))
234
+ ax.axis('off')
235
+ info_lines = [
236
+ f"Model: {state.get('algorithm', 'MLP')}",
237
+ f"Task: {state.get('task_type', 'Unknown')}",
238
+ f"Features: {len(feature_cols)}",
239
+ ]
240
+ if hasattr(model, 'hidden_layer_sizes'):
241
+ info_lines.append(f"Hidden Layers: {model.hidden_layer_sizes}")
242
+ if hasattr(model, 'activation'):
243
+ info_lines.append(f"Activation: {model.activation}")
244
+ if hasattr(model, 'n_iter_'):
245
+ info_lines.append(f"Epochs Trained: {model.n_iter_}")
246
+ metrics = state.get('metrics', {})
247
+ if metrics:
248
+ info_lines.append("")
249
+ info_lines.append("Metrics:")
250
+ for k, v in metrics.items():
251
+ if isinstance(v, float):
252
+ info_lines.append(f" {k}: {v:.4f}")
253
+ text = "\n".join(info_lines)
254
+ ax.text(0.1, 0.9, text, transform=ax.transAxes, fontsize=12,
255
+ verticalalignment='top', fontfamily='monospace',
256
+ bbox=dict(boxstyle='round', facecolor='#F0F9FF', alpha=0.8))
257
+ ax.set_title('Deep Learning Model Summary', fontsize=14, fontweight='bold')
258
+ plt.tight_layout()
259
+ path = os.path.join(output_dir, 'dl_model_summary.png')
260
+ plt.savefig(path, dpi=150, bbox_inches='tight')
261
+ plt.close()
262
+ charts.append(path)
263
+ print(f" ✅ DL Model Summary → {path}")
264
+ except Exception as e:
265
+ print(f" ⚠️ DL Model Summary failed: {e}")
266
+
267
+ # =====================================================================
268
+ # CLASSIFICATION-SPECIFIC CHARTS
269
+ # =====================================================================
270
+ if is_clf and y_test is not None and y_pred is not None:
271
+ n_classes = len(np.unique(y_test))
272
+ is_binary = n_classes == 2
273
+
274
+ # --- 3. CONFUSION MATRIX (normalized + counts) ---
275
+ try:
276
+ cm = confusion_matrix(y_test, y_pred)
277
+ cm_norm = cm.astype('float') / cm.sum(axis=1, keepdims=True)
278
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
279
+
280
+ fig, axes = plt.subplots(1, 2, figsize=(14, 5))
281
+ # Counts
282
+ sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
283
+ xticklabels=labels, yticklabels=labels)
284
+ axes[0].set_title('Confusion Matrix (Counts)')
285
+ axes[0].set_xlabel('Predicted')
286
+ axes[0].set_ylabel('Actual')
287
+ # Normalized
288
+ sns.heatmap(cm_norm, annot=True, fmt='.2%', cmap='Blues', ax=axes[1],
289
+ xticklabels=labels, yticklabels=labels)
290
+ axes[1].set_title('Confusion Matrix (Normalized)')
291
+ axes[1].set_xlabel('Predicted')
292
+ axes[1].set_ylabel('Actual')
293
+ plt.tight_layout()
294
+ path = os.path.join(output_dir, 'confusion_matrix.png')
295
+ plt.savefig(path, dpi=150, bbox_inches='tight')
296
+ plt.close()
297
+ charts.append(path)
298
+ print(f" ✅ Confusion Matrix → {path}")
299
+ except Exception as e:
300
+ print(f" ⚠️ Confusion Matrix failed: {e}")
301
+
302
+ # --- 4. CLASS DISTRIBUTION (actual vs predicted) ---
303
+ try:
304
+ fig, axes = plt.subplots(1, 2, figsize=(12, 5))
305
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
306
+ # Actual
307
+ actual_counts = pd.Series(y_test).value_counts().sort_index()
308
+ pred_counts = pd.Series(y_pred).value_counts().sort_index()
309
+ axes[0].bar(range(len(actual_counts)), actual_counts.values, color=sns.color_palette('Set2'))
310
+ axes[0].set_xticks(range(len(actual_counts)))
311
+ axes[0].set_xticklabels(labels, rotation=45, ha='right')
312
+ axes[0].set_title('Actual Distribution')
313
+ axes[0].set_ylabel('Count')
314
+ axes[1].bar(range(len(pred_counts)), pred_counts.reindex(actual_counts.index, fill_value=0).values,
315
+ color=sns.color_palette('Set3'))
316
+ axes[1].set_xticks(range(len(actual_counts)))
317
+ axes[1].set_xticklabels(labels, rotation=45, ha='right')
318
+ axes[1].set_title('Predicted Distribution')
319
+ axes[1].set_ylabel('Count')
320
+ plt.suptitle('Class Distribution: Actual vs Predicted', fontsize=14)
321
+ plt.tight_layout()
322
+ path = os.path.join(output_dir, 'class_distribution.png')
323
+ plt.savefig(path, dpi=150, bbox_inches='tight')
324
+ plt.close()
325
+ charts.append(path)
326
+ print(f" ✅ Class Distribution → {path}")
327
+ except Exception as e:
328
+ print(f" ⚠️ Class Distribution failed: {e}")
329
+
330
+ # --- 5. ROC CURVE (binary + multi-class) ---
331
+ if y_proba is not None:
332
+ try:
333
+ fig, ax = plt.subplots(figsize=(8, 8))
334
+ if is_binary:
335
+ proba_pos = y_proba[:, 1] if y_proba.ndim == 2 else y_proba
336
+ fpr, tpr, _ = roc_curve(y_test, proba_pos)
337
+ roc_auc = auc(fpr, tpr)
338
+ ax.plot(fpr, tpr, color='#4F46E5', lw=2, label=f'ROC (AUC = {roc_auc:.4f})')
339
+ else:
340
+ y_test_bin = label_binarize(y_test, classes=sorted(np.unique(y_test)))
341
+ colors = plt.cm.Set1(np.linspace(0, 1, n_classes))
342
+ for i in range(min(n_classes, y_proba.shape[1])):
343
+ fpr, tpr, _ = roc_curve(y_test_bin[:, i], y_proba[:, i])
344
+ roc_auc = auc(fpr, tpr)
345
+ label = class_names[i] if class_names and i < len(class_names) else f'Class {i}'
346
+ ax.plot(fpr, tpr, color=colors[i], lw=2, label=f'{label} (AUC={roc_auc:.3f})')
347
+ ax.plot([0, 1], [0, 1], 'k--', lw=1, alpha=0.5)
348
+ ax.set_xlabel('False Positive Rate')
349
+ ax.set_ylabel('True Positive Rate')
350
+ ax.set_title('ROC Curve')
351
+ ax.legend(loc='lower right')
352
+ ax.set_xlim([0, 1])
353
+ ax.set_ylim([0, 1.02])
354
+ plt.tight_layout()
355
+ path = os.path.join(output_dir, 'roc_curve.png')
356
+ plt.savefig(path, dpi=150, bbox_inches='tight')
357
+ plt.close()
358
+ charts.append(path)
359
+ print(f" ✅ ROC Curve → {path}")
360
+ except Exception as e:
361
+ print(f" ⚠️ ROC Curve failed: {e}")
362
+
363
+ # --- 6. PRECISION-RECALL CURVE (binary) ---
364
+ if y_proba is not None and is_binary:
365
+ try:
366
+ proba_pos = y_proba[:, 1] if y_proba.ndim == 2 else y_proba
367
+ precision_vals, recall_vals, _ = precision_recall_curve(y_test, proba_pos)
368
+ ap = average_precision_score(y_test, proba_pos)
369
+ fig, ax = plt.subplots(figsize=(8, 6))
370
+ ax.plot(recall_vals, precision_vals, color='#059669', lw=2,
371
+ label=f'AP = {ap:.4f}')
372
+ ax.set_xlabel('Recall')
373
+ ax.set_ylabel('Precision')
374
+ ax.set_title('Precision-Recall Curve')
375
+ ax.legend(loc='lower left')
376
+ ax.set_xlim([0, 1])
377
+ ax.set_ylim([0, 1.02])
378
+ plt.tight_layout()
379
+ path = os.path.join(output_dir, 'precision_recall.png')
380
+ plt.savefig(path, dpi=150, bbox_inches='tight')
381
+ plt.close()
382
+ charts.append(path)
383
+ print(f" ✅ Precision-Recall → {path}")
384
+ except Exception as e:
385
+ print(f" ⚠️ Precision-Recall failed: {e}")
386
+
387
+ # --- 7. CALIBRATION CURVE (binary) ---
388
+ if y_proba is not None and is_binary and HAS_CALIBRATION and len(y_test) >= 20:
389
+ try:
390
+ proba_pos = y_proba[:, 1] if y_proba.ndim == 2 else y_proba
391
+ fraction_of_positives, mean_predicted_value = calibration_curve(
392
+ y_test, proba_pos, n_bins=10
393
+ )
394
+ fig, ax = plt.subplots(figsize=(8, 6))
395
+ ax.plot(mean_predicted_value, fraction_of_positives, 's-', color='#4F46E5',
396
+ label='Model', lw=2)
397
+ ax.plot([0, 1], [0, 1], 'k--', label='Perfectly calibrated')
398
+ ax.set_xlabel('Mean Predicted Probability')
399
+ ax.set_ylabel('Fraction of Positives')
400
+ ax.set_title('Calibration Curve')
401
+ ax.legend()
402
+ plt.tight_layout()
403
+ path = os.path.join(output_dir, 'calibration.png')
404
+ plt.savefig(path, dpi=150, bbox_inches='tight')
405
+ plt.close()
406
+ charts.append(path)
407
+ print(f" ✅ Calibration Curve → {path}")
408
+ except Exception as e:
409
+ print(f" ⚠️ Calibration Curve failed: {e}")
410
+
411
+ # --- 8. CONFIDENCE HISTOGRAM ---
412
+ if y_proba is not None:
413
+ try:
414
+ max_proba = np.max(y_proba, axis=1) if y_proba.ndim == 2 else y_proba
415
+ correct = (y_test == y_pred)
416
+ fig, ax = plt.subplots(figsize=(8, 5))
417
+ ax.hist(max_proba[correct], bins=20, alpha=0.7, color='#22C55E', label='Correct', edgecolor='white')
418
+ ax.hist(max_proba[~correct], bins=20, alpha=0.7, color='#EF4444', label='Incorrect', edgecolor='white')
419
+ ax.set_xlabel('Prediction Confidence')
420
+ ax.set_ylabel('Count')
421
+ ax.set_title('Confidence Distribution: Correct vs Incorrect')
422
+ ax.legend()
423
+ plt.tight_layout()
424
+ path = os.path.join(output_dir, 'confidence_histogram.png')
425
+ plt.savefig(path, dpi=150, bbox_inches='tight')
426
+ plt.close()
427
+ charts.append(path)
428
+ print(f" ✅ Confidence Histogram → {path}")
429
+ except Exception as e:
430
+ print(f" ⚠️ Confidence Histogram failed: {e}")
431
+
432
+ # --- 9. METRICS HEATMAP (per-class precision/recall/f1) ---
433
+ if n_classes <= 10:
434
+ try:
435
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
436
+ prec_per_class = precision_score(y_test, y_pred, average=None, zero_division=0)
437
+ rec_per_class = recall_score(y_test, y_pred, average=None, zero_division=0)
438
+ f1_per_class = f1_score(y_test, y_pred, average=None, zero_division=0)
439
+
440
+ metrics_df = pd.DataFrame({
441
+ 'Precision': prec_per_class[:len(labels)],
442
+ 'Recall': rec_per_class[:len(labels)],
443
+ 'F1-Score': f1_per_class[:len(labels)],
444
+ }, index=labels[:len(prec_per_class)])
445
+
446
+ fig, ax = plt.subplots(figsize=(8, max(3, len(labels) * 0.5)))
447
+ sns.heatmap(metrics_df, annot=True, fmt='.3f', cmap='YlGn', ax=ax,
448
+ vmin=0, vmax=1, linewidths=0.5)
449
+ ax.set_title('Per-Class Metrics Heatmap')
450
+ plt.tight_layout()
451
+ path = os.path.join(output_dir, 'metrics_heatmap.png')
452
+ plt.savefig(path, dpi=150, bbox_inches='tight')
453
+ plt.close()
454
+ charts.append(path)
455
+ print(f" ✅ Metrics Heatmap → {path}")
456
+ except Exception as e:
457
+ print(f" ⚠️ Metrics Heatmap failed: {e}")
458
+
459
+ # --- 10. CLASS BALANCE PIE CHART (actual vs predicted) ---
460
+ try:
461
+ fig, axes = plt.subplots(1, 2, figsize=(12, 5))
462
+ labels = class_names if class_names else sorted(np.unique(y_test).tolist())
463
+ actual_counts = pd.Series(y_test).value_counts().sort_index()
464
+ pred_counts = pd.Series(y_pred).value_counts().sort_index()
465
+ colors_pie = sns.color_palette('Set2', len(labels))
466
+ axes[0].pie(actual_counts.values, labels=[str(l)[:15] for l in labels],
467
+ autopct='%1.1f%%', colors=colors_pie,
468
+ wedgeprops={'edgecolor': 'white'})
469
+ axes[0].set_title('Actual Class Balance')
470
+ axes[1].pie(pred_counts.reindex(actual_counts.index, fill_value=0).values,
471
+ labels=[str(l)[:15] for l in labels],
472
+ autopct='%1.1f%%', colors=colors_pie,
473
+ wedgeprops={'edgecolor': 'white'})
474
+ axes[1].set_title('Predicted Class Balance')
475
+ plt.suptitle('Class Balance: Actual vs Predicted', fontsize=14)
476
+ plt.tight_layout()
477
+ path = os.path.join(output_dir, 'class_balance_pie.png')
478
+ plt.savefig(path, dpi=150, bbox_inches='tight')
479
+ plt.close()
480
+ charts.append(path)
481
+ print(f" ✅ Class Balance Pie → {path}")
482
+ except Exception as e:
483
+ print(f" ⚠️ Class Balance Pie failed: {e}")
484
+
485
+ # --- 11. CONFIDENCE BOX PLOT BY CLASS ---
486
+ if y_proba is not None:
487
+ try:
488
+ fig, ax = plt.subplots(figsize=(10, 6))
489
+ max_proba = np.max(y_proba, axis=1) if y_proba.ndim == 2 else y_proba
490
+ classes = sorted(np.unique(y_test).tolist())
491
+ data_boxes = [max_proba[y_test == c] for c in classes]
492
+ bp = ax.boxplot(data_boxes, patch_artist=True)
493
+ box_colors = sns.color_palette('Set2', len(classes))
494
+ for i, patch in enumerate(bp['boxes']):
495
+ patch.set_facecolor(box_colors[i % len(box_colors)])
496
+ patch.set_alpha(0.7)
497
+ labels = [str(class_names[i])[:15] if class_names and i < len(class_names) else str(c) for i, c in enumerate(classes)]
498
+ ax.set_xticklabels(labels, rotation=45, ha='right')
499
+ ax.set_xlabel('Class')
500
+ ax.set_ylabel('Prediction Confidence')
501
+ ax.set_title('Confidence Distribution by Class')
502
+ plt.tight_layout()
503
+ path = os.path.join(output_dir, 'confidence_boxplot.png')
504
+ plt.savefig(path, dpi=150, bbox_inches='tight')
505
+ plt.close()
506
+ charts.append(path)
507
+ print(f" ✅ Confidence Boxplot → {path}")
508
+ except Exception as e:
509
+ print(f" ⚠️ Confidence Boxplot failed: {e}")
510
+
511
+ # --- 12. SCORE DISTRIBUTION (KDE per class) ---
512
+ if y_proba is not None and y_proba.ndim == 2:
513
+ try:
514
+ fig, ax = plt.subplots(figsize=(10, 6))
515
+ for i in range(min(y_proba.shape[1], 5)):
516
+ label = str(class_names[i])[:15] if class_names and i < len(class_names) else f'Class {i}'
517
+ try:
518
+ sns.kdeplot(y_proba[:, i], ax=ax, label=label, lw=2)
519
+ except:
520
+ ax.hist(y_proba[:, i], bins=30, alpha=0.5, label=label, density=True)
521
+ ax.set_xlabel('Prediction Score')
522
+ ax.set_ylabel('Density')
523
+ ax.set_title('Score Distribution by Class')
524
+ ax.legend()
525
+ plt.tight_layout()
526
+ path = os.path.join(output_dir, 'score_distribution.png')
527
+ plt.savefig(path, dpi=150, bbox_inches='tight')
528
+ plt.close()
529
+ charts.append(path)
530
+ print(f" ✅ Score Distribution → {path}")
531
+ except Exception as e:
532
+ print(f" ⚠️ Score Distribution failed: {e}")
533
+
534
+ # =====================================================================
535
+ # REGRESSION-SPECIFIC CHARTS
536
+ # =====================================================================
537
+ if not is_clf and y_test is not None and y_pred is not None:
538
+
539
+ # --- 3. ACTUAL VS PREDICTED ---
540
+ try:
541
+ r2 = r2_score(y_test, y_pred)
542
+ mae = mean_absolute_error(y_test, y_pred)
543
+ rmse = np.sqrt(mean_squared_error(y_test, y_pred))
544
+
545
+ fig, ax = plt.subplots(figsize=(8, 8))
546
+ ax.scatter(y_test, y_pred, alpha=0.5, color='#3B82F6', edgecolors='white', s=50)
547
+ lims = [min(y_test.min(), y_pred.min()), max(y_test.max(), y_pred.max())]
548
+ ax.plot(lims, lims, '--', color='#EF4444', linewidth=2, label='Perfect')
549
+ ax.set_title(f'Actual vs Predicted — R²={r2:.4f}, MAE={mae:.4f}, RMSE={rmse:.4f}')
550
+ ax.set_xlabel('Actual')
551
+ ax.set_ylabel('Predicted')
552
+ ax.legend()
553
+ plt.tight_layout()
554
+ path = os.path.join(output_dir, 'actual_vs_predicted.png')
555
+ plt.savefig(path, dpi=150, bbox_inches='tight')
556
+ plt.close()
557
+ charts.append(path)
558
+ print(f" ✅ Actual vs Predicted → {path}")
559
+ except Exception as e:
560
+ print(f" ⚠️ Actual vs Predicted failed: {e}")
561
+
562
+ # --- 4. RESIDUALS ANALYSIS (2x2 grid) ---
563
+ try:
564
+ residuals = y_test - y_pred
565
+ fig, axes = plt.subplots(2, 2, figsize=(12, 10))
566
+ # Residuals vs Predicted
567
+ axes[0, 0].scatter(y_pred, residuals, alpha=0.5, color='#8B5CF6', s=30)
568
+ axes[0, 0].axhline(y=0, color='#EF4444', linestyle='--', linewidth=2)
569
+ axes[0, 0].set_title('Residuals vs Predicted')
570
+ axes[0, 0].set_xlabel('Predicted'); axes[0, 0].set_ylabel('Residuals')
571
+ # Histogram
572
+ axes[0, 1].hist(residuals, bins=30, color='#06B6D4', edgecolor='white', alpha=0.8)
573
+ axes[0, 1].set_title('Residual Distribution')
574
+ axes[0, 1].set_xlabel('Residual'); axes[0, 1].set_ylabel('Frequency')
575
+ # Q-Q plot
576
+ sorted_res = np.sort(residuals)
577
+ n = len(sorted_res)
578
+ theoretical = np.random.normal(0, 1, n)
579
+ theoretical.sort()
580
+ axes[1, 0].scatter(theoretical, sorted_res, alpha=0.5, color='#F59E0B', s=20)
581
+ lims = [min(theoretical.min(), sorted_res.min()), max(theoretical.max(), sorted_res.max())]
582
+ axes[1, 0].plot(lims, lims, 'r--')
583
+ axes[1, 0].set_title('Q-Q Plot')
584
+ axes[1, 0].set_xlabel('Theoretical Quantiles'); axes[1, 0].set_ylabel('Sample Quantiles')
585
+ # Residuals vs Index
586
+ axes[1, 1].plot(range(len(residuals)), residuals, 'o', alpha=0.5, color='#10B981', markersize=4)
587
+ axes[1, 1].axhline(y=0, color='#EF4444', linestyle='--')
588
+ axes[1, 1].set_title('Residuals vs Index')
589
+ axes[1, 1].set_xlabel('Sample Index'); axes[1, 1].set_ylabel('Residual')
590
+ plt.suptitle('Residuals Analysis', fontsize=14)
591
+ plt.tight_layout()
592
+ path = os.path.join(output_dir, 'residuals_analysis.png')
593
+ plt.savefig(path, dpi=150, bbox_inches='tight')
594
+ plt.close()
595
+ charts.append(path)
596
+ print(f" ✅ Residuals Analysis → {path}")
597
+ except Exception as e:
598
+ print(f" ⚠️ Residuals Analysis failed: {e}")
599
+
600
+ # --- 5. ERROR DISTRIBUTION ---
601
+ try:
602
+ abs_errors = np.abs(y_test - y_pred)
603
+ fig, ax = plt.subplots(figsize=(8, 5))
604
+ ax.hist(abs_errors, bins=30, color='#F97316', edgecolor='white', alpha=0.8, density=True)
605
+ p50 = np.percentile(abs_errors, 50)
606
+ p90 = np.percentile(abs_errors, 90)
607
+ p95 = np.percentile(abs_errors, 95)
608
+ ax.axvline(p50, color='green', linestyle='--', label=f'Median: {p50:.3f}')
609
+ ax.axvline(p90, color='orange', linestyle='--', label=f'P90: {p90:.3f}')
610
+ ax.axvline(p95, color='red', linestyle='--', label=f'P95: {p95:.3f}')
611
+ ax.set_title('Absolute Error Distribution')
612
+ ax.set_xlabel('Absolute Error')
613
+ ax.set_ylabel('Density')
614
+ ax.legend()
615
+ plt.tight_layout()
616
+ path = os.path.join(output_dir, 'error_distribution.png')
617
+ plt.savefig(path, dpi=150, bbox_inches='tight')
618
+ plt.close()
619
+ charts.append(path)
620
+ print(f" ✅ Error Distribution → {path}")
621
+ except Exception as e:
622
+ print(f" ⚠️ Error Distribution failed: {e}")
623
+
624
+ # --- 6. PREDICTION OVERVIEW (sorted with ±2sigma) ---
625
+ try:
626
+ idx_sorted = np.argsort(y_test)
627
+ fig, ax = plt.subplots(figsize=(10, 5))
628
+ ax.plot(range(len(y_test)), y_test[idx_sorted], 'o', color='#3B82F6', markersize=3, label='Actual', alpha=0.7)
629
+ ax.plot(range(len(y_pred)), y_pred[idx_sorted], 'x', color='#EF4444', markersize=3, label='Predicted', alpha=0.7)
630
+ residuals_sorted = y_test[idx_sorted] - y_pred[idx_sorted]
631
+ sigma = np.std(residuals_sorted)
632
+ ax.fill_between(range(len(y_test)), y_pred[idx_sorted] - 2*sigma, y_pred[idx_sorted] + 2*sigma,
633
+ alpha=0.15, color='red', label=f'±2σ ({sigma:.3f})')
634
+ ax.set_title('Prediction Overview (Sorted)')
635
+ ax.set_xlabel('Sorted Sample Index')
636
+ ax.set_ylabel('Value')
637
+ ax.legend()
638
+ plt.tight_layout()
639
+ path = os.path.join(output_dir, 'prediction_overview.png')
640
+ plt.savefig(path, dpi=150, bbox_inches='tight')
641
+ plt.close()
642
+ charts.append(path)
643
+ print(f" ✅ Prediction Overview → {path}")
644
+ except Exception as e:
645
+ print(f" ⚠️ Prediction Overview failed: {e}")
646
+
647
+ # --- 7. ERROR BOX PLOT BY QUANTILES ---
648
+ try:
649
+ abs_errors = np.abs(y_test - y_pred)
650
+ quantiles = np.percentile(y_test, [0, 25, 50, 75, 100])
651
+ groups = []
652
+ q_labels = []
653
+ for qi in range(len(quantiles) - 1):
654
+ mask = (y_test >= quantiles[qi]) & (y_test < quantiles[qi+1])
655
+ if mask.sum() > 0:
656
+ groups.append(abs_errors[mask])
657
+ q_labels.append(f'Q{qi+1}')
658
+ if groups:
659
+ fig, ax = plt.subplots(figsize=(10, 6))
660
+ bp = ax.boxplot(groups, patch_artist=True)
661
+ box_colors = ['#3B82F6', '#22C55E', '#F59E0B', '#EF4444']
662
+ for i, patch in enumerate(bp['boxes']):
663
+ patch.set_facecolor(box_colors[i % len(box_colors)])
664
+ patch.set_alpha(0.7)
665
+ ax.set_xticklabels(q_labels)
666
+ ax.set_xlabel('Target Quantile')
667
+ ax.set_ylabel('Absolute Error')
668
+ ax.set_title('Error Distribution by Target Range')
669
+ plt.tight_layout()
670
+ path = os.path.join(output_dir, 'error_boxplot.png')
671
+ plt.savefig(path, dpi=150, bbox_inches='tight')
672
+ plt.close()
673
+ charts.append(path)
674
+ print(f" ✅ Error Boxplot → {path}")
675
+ except Exception as e:
676
+ print(f" ⚠️ Error Boxplot failed: {e}")
677
+
678
+ # --- 8. SCALE-LOCATION PLOT ---
679
+ try:
680
+ residuals_sl = y_test - y_pred
681
+ sqrt_abs_resid = np.sqrt(np.abs(residuals_sl))
682
+ fig, ax = plt.subplots(figsize=(8, 6))
683
+ ax.scatter(y_pred, sqrt_abs_resid, alpha=0.5, color='#8B5CF6', s=30)
684
+ z = np.polyfit(y_pred, sqrt_abs_resid, 1)
685
+ p = np.poly1d(z)
686
+ ax.plot(np.sort(y_pred), p(np.sort(y_pred)), 'r--', lw=2)
687
+ ax.set_xlabel('Fitted Values')
688
+ ax.set_ylabel('√|Residuals|')
689
+ ax.set_title('Scale-Location Plot')
690
+ plt.tight_layout()
691
+ path = os.path.join(output_dir, 'scale_location.png')
692
+ plt.savefig(path, dpi=150, bbox_inches='tight')
693
+ plt.close()
694
+ charts.append(path)
695
+ print(f" ✅ Scale-Location → {path}")
696
+ except Exception as e:
697
+ print(f" ⚠️ Scale-Location failed: {e}")
698
+
699
+ # =====================================================================
700
+ # DATA EXPLORATION CHARTS (both classification & regression)
701
+ # =====================================================================
702
+ if df is not None:
703
+ y = df[target_col] if target_col in df.columns else None
704
+
705
+ # --- 10. CORRELATION HEATMAP ---
706
+ try:
707
+ numeric_df = df.select_dtypes(include=[np.number])
708
+ if len(numeric_df.columns) >= 2:
709
+ corr = numeric_df.corr()
710
+ fig, ax = plt.subplots(figsize=(min(14, len(corr) + 2), min(12, len(corr) + 1)))
711
+ mask = np.triu(np.ones_like(corr, dtype=bool))
712
+ sns.heatmap(corr, mask=mask, annot=len(corr) <= 15, fmt='.2f',
713
+ cmap='RdBu_r', center=0, ax=ax, square=True, linewidths=0.5)
714
+ ax.set_title('Feature Correlation Heatmap')
715
+ plt.tight_layout()
716
+ path = os.path.join(output_dir, 'correlation_heatmap.png')
717
+ plt.savefig(path, dpi=150, bbox_inches='tight')
718
+ plt.close()
719
+ charts.append(path)
720
+ print(f" ✅ Correlation Heatmap → {path}")
721
+ except Exception as e:
722
+ print(f" ⚠️ Correlation Heatmap failed: {e}")
723
+
724
+ # --- 11. FEATURE DISTRIBUTIONS ---
725
+ try:
726
+ plot_cols = NUMERIC_FEATURES if NUMERIC_FEATURES else df.select_dtypes(include=[np.number]).columns[:9].tolist()
727
+ cols_to_plot = [c for c in plot_cols if c in df.columns][:9]
728
+ if cols_to_plot:
729
+ ncols = min(3, len(cols_to_plot))
730
+ nrows = (len(cols_to_plot) + ncols - 1) // ncols
731
+ fig, axes = plt.subplots(nrows, ncols, figsize=(5 * ncols, 4 * nrows))
732
+ if nrows * ncols == 1:
733
+ axes = np.array([axes])
734
+ axes = axes.flatten()
735
+ for i, col in enumerate(cols_to_plot):
736
+ axes[i].hist(df[col].dropna(), bins=30, color='#4F46E5', edgecolor='white', alpha=0.8)
737
+ axes[i].set_title(col, fontsize=10)
738
+ axes[i].tick_params(labelsize=8)
739
+ for i in range(len(cols_to_plot), len(axes)):
740
+ axes[i].set_visible(False)
741
+ plt.suptitle('Feature Distributions', fontsize=14)
742
+ plt.tight_layout()
743
+ path = os.path.join(output_dir, 'distribution_grid.png')
744
+ plt.savefig(path, dpi=150, bbox_inches='tight')
745
+ plt.close()
746
+ charts.append(path)
747
+ print(f" ✅ Distribution Grid → {path}")
748
+ except Exception as e:
749
+ print(f" ⚠️ Distribution Grid failed: {e}")
750
+
751
+ # --- 12. BOXPLOT GRID ---
752
+ try:
753
+ plot_cols = NUMERIC_FEATURES if NUMERIC_FEATURES else df.select_dtypes(include=[np.number]).columns[:9].tolist()
754
+ cols_to_plot = [c for c in plot_cols if c in df.columns][:9]
755
+ if cols_to_plot:
756
+ ncols = min(3, len(cols_to_plot))
757
+ nrows = (len(cols_to_plot) + ncols - 1) // ncols
758
+ fig, axes = plt.subplots(nrows, ncols, figsize=(5 * ncols, 4 * nrows))
759
+ if nrows * ncols == 1:
760
+ axes = np.array([axes])
761
+ axes = axes.flatten()
762
+ colors = sns.color_palette('Set2', len(cols_to_plot))
763
+ for i, col in enumerate(cols_to_plot):
764
+ axes[i].boxplot(df[col].dropna(), patch_artist=True,
765
+ boxprops=dict(facecolor=colors[i], alpha=0.7))
766
+ axes[i].set_title(col, fontsize=10)
767
+ axes[i].tick_params(labelsize=8)
768
+ for i in range(len(cols_to_plot), len(axes)):
769
+ axes[i].set_visible(False)
770
+ plt.suptitle('Feature Boxplots (Outlier Detection)', fontsize=14)
771
+ plt.tight_layout()
772
+ path = os.path.join(output_dir, 'boxplot_grid.png')
773
+ plt.savefig(path, dpi=150, bbox_inches='tight')
774
+ plt.close()
775
+ charts.append(path)
776
+ print(f" ✅ Boxplot Grid → {path}")
777
+ except Exception as e:
778
+ print(f" ⚠️ Boxplot Grid failed: {e}")
779
+
780
+ # --- 13. TARGET DISTRIBUTION (from raw data) ---
781
+ if y is not None:
782
+ try:
783
+ fig, ax = plt.subplots(figsize=(8, 5))
784
+ if is_clf:
785
+ y.value_counts().plot(kind='bar', ax=ax, color=sns.color_palette('Set2'))
786
+ ax.set_ylabel('Count')
787
+ else:
788
+ ax.hist(y.dropna(), bins=30, color='#06B6D4', edgecolor='white', alpha=0.8)
789
+ ax.set_ylabel('Frequency')
790
+ ax.set_title(f'Target Distribution: price_range')
791
+ ax.set_xlabel('price_range')
792
+ plt.xticks(rotation=45, ha='right')
793
+ plt.tight_layout()
794
+ path = os.path.join(output_dir, 'target_distribution.png')
795
+ plt.savefig(path, dpi=150, bbox_inches='tight')
796
+ plt.close()
797
+ charts.append(path)
798
+ print(f" ✅ Target Distribution → {path}")
799
+ except Exception as e:
800
+ print(f" ⚠️ Target Distribution failed: {e}")
801
+
802
+ # =====================================================================
803
+ # UNSUPERVISED LEARNING — Auto Clustering (KMeans)
804
+ # =====================================================================
805
+ try:
806
+ from sklearn.cluster import KMeans
807
+ from sklearn.preprocessing import StandardScaler as ClusterScaler
808
+ from sklearn.decomposition import PCA
809
+ from sklearn.metrics import silhouette_score as sil_score
810
+
811
+ numeric_df = df.select_dtypes(include=[np.number]).dropna()
812
+ if len(numeric_df) >= 10 and len(numeric_df.columns) >= 2:
813
+ # Scale for clustering
814
+ c_scaler = ClusterScaler()
815
+ X_clust = c_scaler.fit_transform(numeric_df)
816
+
817
+ # Find optimal k using silhouette score
818
+ best_k = 3
819
+ best_sil = -1
820
+ for k in range(2, min(8, len(numeric_df))):
821
+ try:
822
+ km = KMeans(n_clusters=k, random_state=42, n_init=10)
823
+ labels_k = km.fit_predict(X_clust)
824
+ sil_k = sil_score(X_clust, labels_k)
825
+ if sil_k > best_sil:
826
+ best_sil = sil_k
827
+ best_k = k
828
+ except:
829
+ pass
830
+
831
+ # Run clustering with best k
832
+ km_final = KMeans(n_clusters=best_k, random_state=42, n_init=10)
833
+ labels = km_final.fit_predict(X_clust)
834
+
835
+ # PCA for 2D visualization
836
+ pca = PCA(n_components=2)
837
+ X_2d = pca.fit_transform(X_clust)
838
+
839
+ # --- Cluster Scatter Plot ---
840
+ fig, ax = plt.subplots(figsize=(10, 8))
841
+ colors_clust = ['#2563eb', '#16a34a', '#dc2626', '#ea580c', '#9333ea',
842
+ '#0891b2', '#db2777', '#d97706', '#0d9488', '#4f46e5']
843
+ for i in range(best_k):
844
+ mask = labels == i
845
+ ax.scatter(X_2d[mask, 0], X_2d[mask, 1],
846
+ c=colors_clust[i % len(colors_clust)],
847
+ label=f'Cluster {i}', alpha=0.7, s=50,
848
+ edgecolors='white', linewidth=0.5)
849
+ ax.set_xlabel('PCA Component 1', fontweight='bold')
850
+ ax.set_ylabel('PCA Component 2', fontweight='bold')
851
+ ax.set_title(f'KMeans Clustering (k={best_k}, Silhouette={best_sil:.3f})',
852
+ fontweight='bold', pad=15, fontsize=14)
853
+ ax.legend(loc='best')
854
+ ax.grid(alpha=0.3)
855
+ plt.tight_layout()
856
+ path = os.path.join(output_dir, 'cluster_scatter.png')
857
+ plt.savefig(path, dpi=150, bbox_inches='tight')
858
+ plt.close()
859
+ charts.append(path)
860
+ print(f" ✅ Cluster Scatter → {path}")
861
+
862
+ # --- Cluster Distribution Bar Chart ---
863
+ fig, ax = plt.subplots(figsize=(8, 5))
864
+ unique, counts = np.unique(labels, return_counts=True)
865
+ ax.bar([f'Cluster {u}' for u in unique], counts,
866
+ color=[colors_clust[u % len(colors_clust)] for u in unique],
867
+ edgecolor='white')
868
+ ax.set_xlabel('Cluster')
869
+ ax.set_ylabel('Count')
870
+ ax.set_title('Cluster Size Distribution')
871
+ for i, (u, c) in enumerate(zip(unique, counts)):
872
+ ax.text(i, c + 0.5, str(c), ha='center', fontsize=10)
873
+ plt.tight_layout()
874
+ path = os.path.join(output_dir, 'cluster_distribution.png')
875
+ plt.savefig(path, dpi=150, bbox_inches='tight')
876
+ plt.close()
877
+ charts.append(path)
878
+ print(f" ✅ Cluster Distribution → {path}")
879
+ except Exception as e:
880
+ print(f" ⚠️ Clustering charts failed: {e}")
881
+
882
+ print(f"\n📊 Generated {len(charts)} charts in '{output_dir}/'")
883
+ return charts
884
+
885
+
886
+ def main():
887
+ parser = argparse.ArgumentParser(description="Generate charts from your trained model")
888
+ parser.add_argument('--model', type=str, default='model.pkl', help='Path to traditional ML model')
889
+ parser.add_argument('--dl-model', type=str, default='deep_learning_model.pkl', help='Path to DL model')
890
+ parser.add_argument('--mode', type=str, default='auto', choices=['auto', 'traditional', 'dl'],
891
+ help='Which model to visualize (auto uses BEST_MODE)')
892
+ parser.add_argument('--data', type=str, default='data/cleaned_data.csv')
893
+ parser.add_argument('--output', type=str, default='charts')
894
+ args = parser.parse_args()
895
+
896
+ print("\n" + "=" * 60)
897
+ print("📊 DataVision Chart Generator")
898
+ print("=" * 60)
899
+
900
+ # Decide which model to load based on BEST_MODE
901
+ use_dl = False
902
+ state = None
903
+ df = None
904
+
905
+ if args.mode == 'dl' or (args.mode == 'auto' and BEST_MODE == 'deep_learning'):
906
+ state = load_dl_model(args.dl_model)
907
+ if state:
908
+ use_dl = True
909
+ else:
910
+ print("⚠️ DL model not found, falling back to traditional ML model...")
911
+
912
+ if not use_dl and args.mode != 'dl':
913
+ state, df = load_model_and_data(args.model, args.data)
914
+ if state is None and os.path.exists(args.dl_model):
915
+ state = load_dl_model(args.dl_model)
916
+ if state:
917
+ use_dl = True
918
+
919
+ if state is None:
920
+ print("❌ No model found. Provide model.pkl or deep_learning_model.pkl")
921
+ return
922
+
923
+ # Load data if not yet loaded
924
+ if df is None:
925
+ data_path = args.data
926
+ df = pd.read_csv(data_path) if os.path.exists(data_path) else None
927
+
928
+ mode_display = 'deep_learning' if use_dl else state.get('mode', 'traditional')
929
+ model_name = state.get('model_name', state.get('algorithm', 'Unknown'))
930
+
931
+ print(f"\n🏆 Model: {model_name}")
932
+ print(f"📊 Mode: {mode_display}")
933
+ print(f"🎯 Task: {state.get('task_type', TASK_TYPE)}")
934
+ if use_dl:
935
+ print(f"🧠 Architecture: {state.get('algorithm', 'MLP')}")
936
+ if df is not None:
937
+ print(f"📋 Data: {len(df)} rows, {len(df.columns)} columns")
938
+
939
+ generate_charts(state, df, args.output)
940
+ print(f"\n✅ Done! Open '{args.output}/' to view charts.")
941
+
942
+
943
+ if __name__ == '__main__':
944
+ main()
backend/__init__.py ADDED
File without changes
backend/agents/__init__.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🤖 Agentic AutoML - Complete Agent Package
3
+
4
+ This package contains all components for the autonomous multi-agent AutoML system.
5
+
6
+ Agents:
7
+ 1. Data Quality Agent - Detects and fixes dataset issues
8
+ 2. Preprocessing Agent - Smart encoding and scaling
9
+ 3. Feature Engineer Agent - Feature creation and selection
10
+ 4. Model Strategy Agent - Algorithm selection
11
+ 5. Hyperparameter Agent - Optuna optimization
12
+ 6. Training Validator Agent - Bias-variance analysis
13
+ 7. Evaluation Agent - Final approval gate
14
+ 8. Visualization Agent - Explainability charts
15
+ 9. Deployment Agent - Production inference
16
+ """
17
+
18
+ # Base components
19
+ from .base import (
20
+ BaseAgent,
21
+ AgentResult,
22
+ AgentStatus,
23
+ AgentMessage,
24
+ MessageType,
25
+ Phase,
26
+ AgentRegistry
27
+ )
28
+
29
+ # Memory and orchestration
30
+ from .memory import AgentMemory, StateEntry, Artifact
31
+ from .orchestrator import AgentOrchestrator, PipelineStatus
32
+
33
+ # Data agents
34
+ from .data_quality import DataQualityAgent
35
+ from .preprocessing import PreprocessingAgent
36
+ from .feature_engineer import FeatureEngineerAgent
37
+
38
+ # Model agents
39
+ from .model_strategy import ModelStrategyAgent
40
+ from .hyperparam import HyperparamAgent
41
+ from .training_validator import TrainingValidatorAgent
42
+
43
+ # Validation agents
44
+ from .evaluation import EvaluationAgent
45
+ from .visualization import VisualizationAgent
46
+ from .deployment import DeploymentAgent
47
+
48
+
49
+ def create_agentic_pipeline() -> AgentOrchestrator:
50
+ """
51
+ Factory function to create a fully configured agentic AutoML pipeline.
52
+
53
+ Returns:
54
+ AgentOrchestrator with all agents registered
55
+ """
56
+ orchestrator = AgentOrchestrator()
57
+
58
+ # Register all agents
59
+ orchestrator.register_agent("data_quality", DataQualityAgent())
60
+ orchestrator.register_agent("preprocessing", PreprocessingAgent())
61
+ orchestrator.register_agent("feature_engineer", FeatureEngineerAgent())
62
+ orchestrator.register_agent("model_strategy", ModelStrategyAgent())
63
+ orchestrator.register_agent("hyperparam", HyperparamAgent())
64
+ orchestrator.register_agent("training_validator", TrainingValidatorAgent())
65
+ orchestrator.register_agent("evaluation", EvaluationAgent())
66
+ orchestrator.register_agent("visualization", VisualizationAgent())
67
+ orchestrator.register_agent("deployment", DeploymentAgent())
68
+
69
+ return orchestrator
70
+
71
+
72
+ __all__ = [
73
+ # Base
74
+ 'BaseAgent',
75
+ 'AgentResult',
76
+ 'AgentStatus',
77
+ 'AgentMessage',
78
+ 'MessageType',
79
+ 'Phase',
80
+ 'AgentRegistry',
81
+
82
+ # Memory
83
+ 'AgentMemory',
84
+ 'StateEntry',
85
+ 'Artifact',
86
+
87
+ # Orchestrator
88
+ 'AgentOrchestrator',
89
+ 'PipelineStatus',
90
+
91
+ # Agents
92
+ 'DataQualityAgent',
93
+ 'PreprocessingAgent',
94
+ 'FeatureEngineerAgent',
95
+ 'ModelStrategyAgent',
96
+ 'HyperparamAgent',
97
+ 'TrainingValidatorAgent',
98
+ 'EvaluationAgent',
99
+ 'VisualizationAgent',
100
+ 'DeploymentAgent',
101
+
102
+ # Factory
103
+ 'create_agentic_pipeline'
104
+ ]
backend/agents/adaptive_rag.py ADDED
@@ -0,0 +1,470 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adaptive RAG Module - Intelligent Query Routing & Retrieval
2
+ """
3
+ Adaptive RAG intelligently selects the best retrieval strategy
4
+ based on query characteristics:
5
+
6
+ 1. Query Classification:
7
+ - Factual: Simple lookups → RAG
8
+ - Analytical: Trends, patterns → GraphRAG
9
+ - Comparative: Multiple entities → Hybrid
10
+ - Exploratory: Open-ended → Graph + Semantic
11
+
12
+ 2. Retrieval Adaptation:
13
+ - Adjusts k (number of results)
14
+ - Adjusts semantic weight for hybrid
15
+ - Selects best data sources
16
+
17
+ 3. Response Quality:
18
+ - Validates response relevance
19
+ - Triggers fallback if low confidence
20
+ """
21
+
22
+ import re
23
+ from typing import Dict, Any, List, Tuple, Optional
24
+ from dataclasses import dataclass
25
+ from enum import Enum
26
+
27
+
28
+ class QueryType(Enum):
29
+ """Query classification types"""
30
+ FACTUAL = "factual" # What, show, list, total
31
+ ANALYTICAL = "analytical" # Why, trend, pattern, correlation
32
+ COMPARATIVE = "comparative" # Compare, vs, difference
33
+ EXPLORATORY = "exploratory" # Insights, analyze, overview
34
+ TEMPORAL = "temporal" # Time-based queries
35
+ ENTITY = "entity" # Specific entity lookup
36
+ AGGREGATE = "aggregate" # Sum, count, average
37
+
38
+
39
+ @dataclass
40
+ class QueryAnalysis:
41
+ """Analysis of user query"""
42
+ query: str
43
+ query_type: QueryType
44
+ confidence: float
45
+ entities: List[str]
46
+ time_references: List[str]
47
+ metrics_mentioned: List[str]
48
+ recommended_route: str
49
+ recommended_k: int
50
+ semantic_weight: float
51
+ requires_graph: bool
52
+ requires_vectors: bool
53
+
54
+
55
+ class AdaptiveRouter:
56
+ """
57
+ Intelligent query routing with adaptive retrieval
58
+ """
59
+
60
+ # Query type patterns
61
+ FACTUAL_PATTERNS = [
62
+ r'\bwhat (is|are|was|were)\b',
63
+ r'\bshow me\b',
64
+ r'\blist\b',
65
+ r'\btotal\b',
66
+ r'\bhow many\b',
67
+ r'\bdisplay\b',
68
+ r'\bget\b',
69
+ r'\blowest\b',
70
+ r'\bhighest\b',
71
+ r'\bleast\b',
72
+ r'\bmost\b',
73
+ r'\bbottom\b',
74
+ r'\bworst\b',
75
+ r'\bbest\b',
76
+ r'\bwho\b',
77
+ ]
78
+
79
+ ANALYTICAL_PATTERNS = [
80
+ r'\bwhy\b',
81
+ r'\bhow did\b',
82
+ r'\btrend\b',
83
+ r'\bpattern\b',
84
+ r'\bcorrelation\b',
85
+ r'\bcause\b',
86
+ r'\breason\b',
87
+ r'\binsight\b',
88
+ r'\banalysis\b',
89
+ r'\bexplain\b',
90
+ ]
91
+
92
+ COMPARATIVE_PATTERNS = [
93
+ r'\bcompare\b',
94
+ r'\bvs\b',
95
+ r'\bversus\b',
96
+ r'\bdifference\b',
97
+ r'\bbetween\b.*\band\b',
98
+ r'\bmore than\b',
99
+ r'\bless than\b',
100
+ ]
101
+
102
+ TEMPORAL_PATTERNS = [
103
+ r'\b(january|february|march|april|may|june|july|august|september|october|november|december)\b',
104
+ r'\b(q1|q2|q3|q4)\b',
105
+ r'\b\d{4}\b', # Year
106
+ r'\blast (week|month|year|quarter)\b',
107
+ r'\bthis (week|month|year|quarter)\b',
108
+ r'\byesterday\b',
109
+ r'\btoday\b',
110
+ ]
111
+
112
+ AGGREGATE_PATTERNS = [
113
+ r'\btotal\b',
114
+ r'\bsum\b',
115
+ r'\baverage\b',
116
+ r'\bcount\b',
117
+ r'\bmaximum\b',
118
+ r'\bminimum\b',
119
+ r'\bmax\b',
120
+ r'\bmin\b',
121
+ ]
122
+
123
+ # Entity patterns
124
+ ENTITY_PATTERNS = [
125
+ r'\bcustomer[:\s]+([A-Za-z0-9_\s]+)',
126
+ r'\bproduct[:\s]+([A-Za-z0-9_\s]+)',
127
+ r'\binvoice[:\s#]+([A-Za-z0-9_]+)',
128
+ ]
129
+
130
+ def __init__(self):
131
+ """Initialize router with compiled patterns"""
132
+ self._compile_patterns()
133
+
134
+ def _compile_patterns(self):
135
+ """Compile regex patterns for efficiency"""
136
+ self.factual_re = [re.compile(p, re.IGNORECASE) for p in self.FACTUAL_PATTERNS]
137
+ self.analytical_re = [re.compile(p, re.IGNORECASE) for p in self.ANALYTICAL_PATTERNS]
138
+ self.comparative_re = [re.compile(p, re.IGNORECASE) for p in self.COMPARATIVE_PATTERNS]
139
+ self.temporal_re = [re.compile(p, re.IGNORECASE) for p in self.TEMPORAL_PATTERNS]
140
+ self.aggregate_re = [re.compile(p, re.IGNORECASE) for p in self.AGGREGATE_PATTERNS]
141
+ self.entity_re = [re.compile(p, re.IGNORECASE) for p in self.ENTITY_PATTERNS]
142
+
143
+ def analyze_query(self, query: str) -> QueryAnalysis:
144
+ """
145
+ Analyze query and determine optimal retrieval strategy
146
+
147
+ Args:
148
+ query: User's query string
149
+
150
+ Returns:
151
+ QueryAnalysis with routing recommendations
152
+ """
153
+ query_lower = query.lower()
154
+
155
+ # Calculate scores for each type
156
+ scores = {
157
+ QueryType.FACTUAL: self._score_patterns(query, self.factual_re),
158
+ QueryType.ANALYTICAL: self._score_patterns(query, self.analytical_re),
159
+ QueryType.COMPARATIVE: self._score_patterns(query, self.comparative_re),
160
+ QueryType.TEMPORAL: self._score_patterns(query, self.temporal_re),
161
+ QueryType.AGGREGATE: self._score_patterns(query, self.aggregate_re),
162
+ }
163
+
164
+ # Determine primary type
165
+ primary_type, confidence = max(scores.items(), key=lambda x: x[1])
166
+
167
+ # Default to analytical if low confidence (encourages graph-first)
168
+ if confidence < 0.3:
169
+ primary_type = QueryType.ANALYTICAL
170
+ confidence = 0.5
171
+
172
+ # Extract entities
173
+ entities = self._extract_entities(query)
174
+
175
+ # Extract time references
176
+ time_refs = self._extract_time_references(query)
177
+
178
+ # Extract metrics
179
+ metrics = self._extract_metrics(query)
180
+
181
+ # Determine route and parameters
182
+ route, k, semantic_weight, requires_graph, requires_vectors = \
183
+ self._determine_strategy(primary_type, entities, time_refs, confidence)
184
+
185
+ return QueryAnalysis(
186
+ query=query,
187
+ query_type=primary_type,
188
+ confidence=confidence,
189
+ entities=entities,
190
+ time_references=time_refs,
191
+ metrics_mentioned=metrics,
192
+ recommended_route=route,
193
+ recommended_k=k,
194
+ semantic_weight=semantic_weight,
195
+ requires_graph=requires_graph,
196
+ requires_vectors=requires_vectors
197
+ )
198
+
199
+ def _score_patterns(self, query: str, patterns: List[re.Pattern]) -> float:
200
+ """Score query against pattern list"""
201
+ matches = sum(1 for p in patterns if p.search(query))
202
+ return min(1.0, matches / 3) # Normalize to 0-1
203
+
204
+ def _extract_entities(self, query: str) -> List[str]:
205
+ """Extract named entities from query"""
206
+ entities = []
207
+ for pattern in self.entity_re:
208
+ matches = pattern.findall(query)
209
+ entities.extend(matches)
210
+ return list(set(entities))
211
+
212
+ def _extract_time_references(self, query: str) -> List[str]:
213
+ """Extract time references from query"""
214
+ refs = []
215
+ for pattern in self.temporal_re:
216
+ matches = pattern.findall(query)
217
+ if matches:
218
+ refs.extend(matches if isinstance(matches[0], str) else [m[0] for m in matches])
219
+ return list(set(refs))
220
+
221
+ def _extract_metrics(self, query: str) -> List[str]:
222
+ """Extract business metrics mentioned"""
223
+ metrics = []
224
+ metric_keywords = [
225
+ 'revenue', 'sales', 'profit', 'margin', 'growth',
226
+ 'cost', 'expense', 'income', 'roi', 'conversion'
227
+ ]
228
+ query_lower = query.lower()
229
+ for metric in metric_keywords:
230
+ if metric in query_lower:
231
+ metrics.append(metric)
232
+ return metrics
233
+
234
+ def _determine_strategy(
235
+ self,
236
+ query_type: QueryType,
237
+ entities: List[str],
238
+ time_refs: List[str],
239
+ confidence: float
240
+ ) -> Tuple[str, int, float, bool, bool]:
241
+ """
242
+ Determine retrieval strategy based on query analysis
243
+
244
+ Returns: (route, k, semantic_weight, requires_graph, requires_vectors)
245
+ """
246
+ # Default: Graph-first architecture
247
+ route = "graph"
248
+ k = 5
249
+ semantic_weight = 0.6
250
+ requires_graph = True
251
+ requires_vectors = True
252
+
253
+ if query_type == QueryType.FACTUAL:
254
+ # Factual queries benefit from RAG
255
+ route = "rag" if not entities else "hybrid"
256
+ k = 6
257
+ semantic_weight = 0.5
258
+ requires_graph = len(entities) > 0
259
+ requires_vectors = True
260
+
261
+ elif query_type == QueryType.ANALYTICAL:
262
+ # Analytical queries need graph reasoning
263
+ route = "graph"
264
+ k = 4
265
+ semantic_weight = 0.7
266
+ requires_graph = True
267
+ requires_vectors = len(time_refs) > 0
268
+
269
+ elif query_type == QueryType.COMPARATIVE:
270
+ # Comparative queries need both
271
+ route = "hybrid"
272
+ k = 8
273
+ semantic_weight = 0.5
274
+ requires_graph = True
275
+ requires_vectors = True
276
+
277
+ elif query_type == QueryType.TEMPORAL:
278
+ # Time-based queries need graph + vectors
279
+ route = "hybrid"
280
+ k = 6
281
+ semantic_weight = 0.6
282
+ requires_graph = True
283
+ requires_vectors = True
284
+
285
+ elif query_type == QueryType.AGGREGATE:
286
+ # Aggregations work best with graph
287
+ route = "graph"
288
+ k = 4
289
+ semantic_weight = 0.5
290
+ requires_graph = True
291
+ requires_vectors = False
292
+
293
+ # Adjust if specific entities mentioned
294
+ if entities:
295
+ k += 2
296
+ requires_graph = True
297
+
298
+ return route, k, semantic_weight, requires_graph, requires_vectors
299
+
300
+
301
+ class AdaptiveRetriever:
302
+ """
303
+ Executes adaptive retrieval based on query analysis
304
+ """
305
+
306
+ def __init__(self, user_id: str):
307
+ """
308
+ Args:
309
+ user_id: User/company identifier
310
+ """
311
+ self.user_id = user_id
312
+ self.router = AdaptiveRouter()
313
+
314
+ def retrieve(self, query: str) -> Dict[str, Any]:
315
+ """
316
+ Perform adaptive retrieval
317
+
318
+ Args:
319
+ query: User query
320
+
321
+ Returns:
322
+ Dictionary with context, analysis, and metadata
323
+ """
324
+ # Analyze query
325
+ analysis = self.router.analyze_query(query)
326
+
327
+ context_parts = []
328
+ sources = []
329
+
330
+ # Get vector context if needed
331
+ if analysis.requires_vectors:
332
+ vector_context = self._get_vector_context(
333
+ query,
334
+ k=analysis.recommended_k,
335
+ alpha=analysis.semantic_weight
336
+ )
337
+ if vector_context:
338
+ context_parts.append(("Documents", vector_context["text"]))
339
+ sources.extend(vector_context["sources"])
340
+
341
+ # Get graph context if needed
342
+ if analysis.requires_graph:
343
+ graph_context = self._get_graph_context(query)
344
+ if graph_context:
345
+ context_parts.append(("Knowledge Graph", graph_context))
346
+
347
+ # Combine context
348
+ combined_context = self._format_context(context_parts)
349
+
350
+ return {
351
+ "context": combined_context,
352
+ "sources": sources,
353
+ "analysis": analysis,
354
+ "route": analysis.recommended_route
355
+ }
356
+
357
+ def _get_vector_context(self, query: str, k: int, alpha: float) -> Optional[Dict]:
358
+ """Get context from vector store using hybrid search"""
359
+ try:
360
+ from vector.hybrid_search import hybrid_retrieve
361
+
362
+ results = hybrid_retrieve(query, k=k, user_id=self.user_id, alpha=alpha)
363
+
364
+ if not results:
365
+ return None
366
+
367
+ texts = []
368
+ sources = []
369
+
370
+ for r in results:
371
+ texts.append(r.get("text", ""))
372
+ source = r.get("source") or r.get("metadata", {}).get("source", "Unknown")
373
+ if source not in sources:
374
+ sources.append(source)
375
+
376
+ return {
377
+ "text": "\n\n".join(texts),
378
+ "sources": sources[:3]
379
+ }
380
+
381
+ except Exception as e:
382
+ print(f"⚠️ Vector retrieval error: {e}")
383
+
384
+ # Fallback to basic retrieval
385
+ try:
386
+ from vector.retriever import retrieve
387
+ results = retrieve(query, k=k, user_id=self.user_id)
388
+
389
+ if results:
390
+ texts = [r.get("text", "") for r in results]
391
+ sources = [r.get("metadata", {}).get("source", "Unknown") for r in results]
392
+ return {
393
+ "text": "\n\n".join(texts),
394
+ "sources": list(set(sources))[:3]
395
+ }
396
+ except:
397
+ pass
398
+
399
+ return None
400
+
401
+ def _get_graph_context(self, query: str) -> Optional[str]:
402
+ """Get context from knowledge graph"""
403
+ try:
404
+ from graph.query import graph_snapshot, revenue_dataframe, get_graph_stats
405
+
406
+ # Get graph statistics
407
+ stats = get_graph_stats(self.user_id)
408
+
409
+ if stats.get("total_nodes", 0) == 0:
410
+ return None
411
+
412
+ # Get graph snapshot
413
+ snapshot = graph_snapshot(self.user_id, max_nodes=50)
414
+
415
+ # Get revenue data summary
416
+ df = revenue_dataframe(self.user_id)
417
+ revenue_summary = ""
418
+
419
+ if df is not None and not df.empty:
420
+ amount_col = 'amount' if 'amount' in df.columns else 'total_amount'
421
+ if amount_col in df.columns:
422
+ total = df[amount_col].sum()
423
+ avg = df[amount_col].mean()
424
+ revenue_summary = f"\nRevenue: Total ${total:,.2f}, Avg ${avg:,.2f}"
425
+
426
+ return f"{snapshot}{revenue_summary}"
427
+
428
+ except Exception as e:
429
+ print(f"⚠️ Graph context error: {e}")
430
+ return None
431
+
432
+ def _format_context(self, parts: List[Tuple[str, str]]) -> str:
433
+ """Format context parts into combined string"""
434
+ if not parts:
435
+ return ""
436
+
437
+ formatted = []
438
+ for title, content in parts:
439
+ formatted.append(f"**{title}:**\n{content}")
440
+
441
+ return "\n\n---\n\n".join(formatted)
442
+
443
+
444
+ def adaptive_retrieve(query: str, user_id: str = "user_001") -> Dict[str, Any]:
445
+ """
446
+ Main adaptive retrieval function
447
+
448
+ Args:
449
+ query: User query
450
+ user_id: User identifier
451
+
452
+ Returns:
453
+ Retrieval results with context and metadata
454
+ """
455
+ retriever = AdaptiveRetriever(user_id)
456
+ return retriever.retrieve(query)
457
+
458
+
459
+ def get_query_analysis(query: str) -> QueryAnalysis:
460
+ """
461
+ Analyze query without retrieval
462
+
463
+ Args:
464
+ query: User query
465
+
466
+ Returns:
467
+ QueryAnalysis object
468
+ """
469
+ router = AdaptiveRouter()
470
+ return router.analyze_query(query)
backend/agents/advanced_charts.py ADDED
@@ -0,0 +1,676 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Advanced Chart Generator - Plotly-Based Dynamic Charts
2
+ """
3
+ Generates Plotly charts based on VisualizationDecision.
4
+
5
+ This module takes decisions from visualization_intelligence.py
6
+ and creates actual chart payloads for the frontend.
7
+
8
+ Features:
9
+ - Dynamic chart type rendering
10
+ - Consistent styling
11
+ - Proper axis labels
12
+ - Interactive tooltips
13
+ - Color schemes per mode
14
+ """
15
+
16
+ from typing import Dict, List, Optional, Any, Tuple
17
+ import json
18
+
19
+
20
+ # Color schemes for different contexts - EXPANDED for 20+ items
21
+ COLOR_SCHEMES = {
22
+ "default": [
23
+ "#f97316", "#3b82f6", "#22c55e", "#a855f7", "#ef4444",
24
+ "#06b6d4", "#f59e0b", "#ec4899", "#8b5cf6", "#14b8a6",
25
+ "#84cc16", "#6366f1", "#f43f5e", "#0ea5e9", "#d946ef",
26
+ "#eab308", "#10b981", "#6b7280", "#78716c", "#0284c7"
27
+ ],
28
+ "executive": ["#1e40af", "#3b82f6", "#60a5fa", "#93c5fd", "#1d4ed8", "#2563eb"],
29
+ "finance": ["#059669", "#10b981", "#34d399", "#6ee7b7", "#047857", "#0d9488"],
30
+ "warning": ["#dc2626", "#ef4444", "#f87171", "#fca5a5", "#b91c1c", "#991b1b"],
31
+ "neutral": ["#6b7280", "#9ca3af", "#d1d5db", "#e5e7eb", "#4b5563", "#374151"],
32
+ }
33
+
34
+
35
+ def generate_dynamic_chart(
36
+ df,
37
+ chart_type: str,
38
+ x_col: str,
39
+ y_col: str,
40
+ title: str,
41
+ group_col: Optional[str] = None,
42
+ currency_symbol: str = "₹",
43
+ color_scheme: str = "default",
44
+ limit: int = 10
45
+ ) -> Dict[str, Any]:
46
+ """
47
+ Generate a Plotly chart payload based on chart type.
48
+
49
+ Returns a JSON-serializable dict for frontend rendering.
50
+ """
51
+ if df is None or df.empty:
52
+ return {"error": "No data available"}
53
+
54
+ colors = COLOR_SCHEMES.get(color_scheme, COLOR_SCHEMES["default"])
55
+
56
+ # Route to specific generator
57
+ generators = {
58
+ "line": _generate_line_chart,
59
+ "bar": _generate_bar_chart,
60
+ "grouped_bar": _generate_grouped_bar_chart,
61
+ "stacked_bar": _generate_stacked_bar_chart,
62
+ "pie": _generate_pie_chart,
63
+ "donut": _generate_donut_chart,
64
+ "area": _generate_area_chart,
65
+ "scatter": _generate_scatter_chart,
66
+ "forecast": _generate_forecast_chart,
67
+ "waterfall": _generate_waterfall_chart,
68
+ }
69
+
70
+ generator = generators.get(chart_type, _generate_bar_chart)
71
+
72
+ return generator(
73
+ df=df,
74
+ x_col=x_col,
75
+ y_col=y_col,
76
+ title=title,
77
+ group_col=group_col,
78
+ currency_symbol=currency_symbol,
79
+ colors=colors,
80
+ limit=limit
81
+ )
82
+
83
+
84
+ def _generate_line_chart(
85
+ df, x_col: str, y_col: str, title: str,
86
+ group_col: Optional[str], currency_symbol: str,
87
+ colors: List[str], limit: int
88
+ ) -> Dict:
89
+ """Generate line chart for trends."""
90
+
91
+ # Aggregate by x column if needed
92
+ if x_col and x_col in df.columns and y_col and y_col in df.columns:
93
+ aggregated = df.groupby(x_col)[y_col].sum().reset_index()
94
+ aggregated = aggregated.sort_values(x_col).tail(limit)
95
+
96
+ x_values = aggregated[x_col].astype(str).tolist()
97
+ y_values = aggregated[y_col].tolist()
98
+ else:
99
+ x_values = list(range(len(df)))
100
+ y_values = df[y_col].tolist() if y_col in df.columns else []
101
+
102
+ return {
103
+ "type": "plotly",
104
+ "chart_type": "line",
105
+ "data": [{
106
+ "type": "scatter",
107
+ "mode": "lines+markers",
108
+ "x": x_values,
109
+ "y": y_values,
110
+ "line": {"color": colors[0], "width": 3},
111
+ "marker": {"size": 8, "color": colors[0]},
112
+ "hovertemplate": f"<b>%{{x}}</b><br>{currency_symbol}%{{y:,.0f}}<extra></extra>"
113
+ }],
114
+ "layout": {
115
+ "title": {"text": title, "font": {"size": 16}},
116
+ "paper_bgcolor": "rgba(0,0,0,0)",
117
+ "plot_bgcolor": "rgba(0,0,0,0)",
118
+ "xaxis": {"title": x_col, "gridcolor": "rgba(128,128,128,0.2)"},
119
+ "yaxis": {"title": y_col, "gridcolor": "rgba(128,128,128,0.2)"},
120
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 50},
121
+ "showlegend": False
122
+ }
123
+ }
124
+
125
+
126
+ def _generate_bar_chart(
127
+ df, x_col: str, y_col: str, title: str,
128
+ group_col: Optional[str], currency_symbol: str,
129
+ colors: List[str], limit: int
130
+ ) -> Dict:
131
+ """Generate bar chart for comparisons/rankings."""
132
+
133
+ if x_col and x_col in df.columns and y_col and y_col in df.columns:
134
+ aggregated = df.groupby(x_col)[y_col].sum().reset_index()
135
+ aggregated = aggregated.sort_values(y_col, ascending=False).head(limit)
136
+
137
+ x_values = aggregated[x_col].astype(str).tolist()
138
+ y_values = aggregated[y_col].tolist()
139
+ else:
140
+ x_values = []
141
+ y_values = []
142
+
143
+ # Assign colors per bar
144
+ bar_colors = [colors[i % len(colors)] for i in range(len(x_values))]
145
+
146
+ return {
147
+ "type": "plotly",
148
+ "chart_type": "bar",
149
+ "data": [{
150
+ "type": "bar",
151
+ "x": x_values,
152
+ "y": y_values,
153
+ "marker": {"color": bar_colors, "line": {"width": 0}},
154
+ "hovertemplate": f"<b>%{{x}}</b><br>{currency_symbol}%{{y:,.0f}}<extra></extra>"
155
+ }],
156
+ "layout": {
157
+ "title": {"text": title, "font": {"size": 16}},
158
+ "paper_bgcolor": "rgba(0,0,0,0)",
159
+ "plot_bgcolor": "rgba(0,0,0,0)",
160
+ "xaxis": {"title": "", "gridcolor": "rgba(128,128,128,0.2)", "tickangle": -45},
161
+ "yaxis": {"title": y_col, "gridcolor": "rgba(128,128,128,0.2)"},
162
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 100},
163
+ "showlegend": False
164
+ }
165
+ }
166
+
167
+
168
+ def _generate_pie_chart(
169
+ df, x_col: str, y_col: str, title: str,
170
+ group_col: Optional[str], currency_symbol: str,
171
+ colors: List[str], limit: int
172
+ ) -> Dict:
173
+ """Generate pie chart for proportional data."""
174
+
175
+ print(f"[PIE CHART DEBUG] x_col={x_col}, y_col={y_col}, limit={limit}")
176
+ print(f"[PIE CHART DEBUG] df columns: {list(df.columns)}")
177
+ print(f"[PIE CHART DEBUG] df['{x_col}'] unique values: {df[x_col].nunique() if x_col in df.columns else 'N/A'}")
178
+ if x_col in df.columns:
179
+ print(f"[PIE CHART DEBUG] Sample values: {df[x_col].head(5).tolist()}")
180
+
181
+ if x_col and x_col in df.columns and y_col and y_col in df.columns:
182
+ aggregated = df.groupby(x_col)[y_col].sum().reset_index()
183
+ aggregated = aggregated.sort_values(y_col, ascending=False).head(limit)
184
+
185
+ labels = aggregated[x_col].astype(str).tolist()
186
+ values = aggregated[y_col].tolist()
187
+ print(f"[PIE CHART DEBUG] Final labels: {labels}")
188
+ else:
189
+ labels = []
190
+ values = []
191
+
192
+ pie_colors = [colors[i % len(colors)] for i in range(len(labels))]
193
+
194
+ return {
195
+ "type": "plotly",
196
+ "chart_type": "pie",
197
+ "data": [{
198
+ "type": "pie",
199
+ "labels": labels,
200
+ "values": values,
201
+ "marker": {"colors": pie_colors},
202
+ "textposition": "auto",
203
+ "textinfo": "label+percent",
204
+ "hovertemplate": f"<b>%{{label}}</b><br>{currency_symbol}%{{value:,.0f}}<br>%{{percent}}<extra></extra>"
205
+ }],
206
+ "layout": {
207
+ "title": {"text": title, "font": {"size": 16}},
208
+ "paper_bgcolor": "rgba(0,0,0,0)",
209
+ "showlegend": True,
210
+ "legend": {"font": {}},
211
+ "margin": {"l": 30, "r": 30, "t": 60, "b": 30}
212
+ }
213
+ }
214
+
215
+
216
+ def _generate_donut_chart(
217
+ df, x_col: str, y_col: str, title: str,
218
+ group_col: Optional[str], currency_symbol: str,
219
+ colors: List[str], limit: int
220
+ ) -> Dict:
221
+ """Generate donut chart (pie with hole)."""
222
+ pie = _generate_pie_chart(df, x_col, y_col, title, group_col, currency_symbol, colors, limit)
223
+ if pie.get("data"):
224
+ pie["data"][0]["hole"] = 0.4
225
+ pie["chart_type"] = "donut"
226
+ return pie
227
+
228
+
229
+ def _generate_grouped_bar_chart(
230
+ df, x_col: str, y_col: str, title: str,
231
+ group_col: Optional[str], currency_symbol: str,
232
+ colors: List[str], limit: int
233
+ ) -> Dict:
234
+ """Generate grouped bar chart for multi-category comparison."""
235
+
236
+ if not group_col or group_col not in df.columns:
237
+ return _generate_bar_chart(df, x_col, y_col, title, None, currency_symbol, colors, limit)
238
+
239
+ traces = []
240
+ groups = df[group_col].unique()[:5] # Max 5 groups
241
+
242
+ for i, group in enumerate(groups):
243
+ group_df = df[df[group_col] == group]
244
+ aggregated = group_df.groupby(x_col)[y_col].sum().reset_index()
245
+ aggregated = aggregated.sort_values(y_col, ascending=False).head(limit)
246
+
247
+ traces.append({
248
+ "type": "bar",
249
+ "name": str(group),
250
+ "x": aggregated[x_col].astype(str).tolist(),
251
+ "y": aggregated[y_col].tolist(),
252
+ "marker": {"color": colors[i % len(colors)]},
253
+ "hovertemplate": f"<b>%{{x}}</b><br>{group}: {currency_symbol}%{{y:,.0f}}<extra></extra>"
254
+ })
255
+
256
+ return {
257
+ "type": "plotly",
258
+ "chart_type": "grouped_bar",
259
+ "data": traces,
260
+ "layout": {
261
+ "title": {"text": title, "font": {"size": 16, "color": "#ffffff"}},
262
+ "paper_bgcolor": "rgba(0,0,0,0)",
263
+ "plot_bgcolor": "rgba(0,0,0,0)",
264
+ "barmode": "group",
265
+ "xaxis": {"title": "", "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
266
+ "yaxis": {"title": y_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
267
+ "legend": {"font": {"color": "#ffffff"}},
268
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 80}
269
+ }
270
+ }
271
+
272
+
273
+ def _generate_stacked_bar_chart(
274
+ df, x_col: str, y_col: str, title: str,
275
+ group_col: Optional[str], currency_symbol: str,
276
+ colors: List[str], limit: int
277
+ ) -> Dict:
278
+ """Generate stacked bar chart for composition."""
279
+
280
+ if not group_col or group_col not in df.columns:
281
+ return _generate_bar_chart(df, x_col, y_col, title, None, currency_symbol, colors, limit)
282
+
283
+ traces = []
284
+ groups = df[group_col].unique()[:8] # Max 8 stacks
285
+
286
+ for i, group in enumerate(groups):
287
+ group_df = df[df[group_col] == group]
288
+ aggregated = group_df.groupby(x_col)[y_col].sum().reset_index()
289
+
290
+ traces.append({
291
+ "type": "bar",
292
+ "name": str(group),
293
+ "x": aggregated[x_col].astype(str).tolist(),
294
+ "y": aggregated[y_col].tolist(),
295
+ "marker": {"color": colors[i % len(colors)]},
296
+ "hovertemplate": f"<b>%{{x}}</b><br>{group}: {currency_symbol}%{{y:,.0f}}<extra></extra>"
297
+ })
298
+
299
+ return {
300
+ "type": "plotly",
301
+ "chart_type": "stacked_bar",
302
+ "data": traces,
303
+ "layout": {
304
+ "title": {"text": title, "font": {"size": 16, "color": "#ffffff"}},
305
+ "paper_bgcolor": "rgba(0,0,0,0)",
306
+ "plot_bgcolor": "rgba(0,0,0,0)",
307
+ "barmode": "stack",
308
+ "xaxis": {"title": "", "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
309
+ "yaxis": {"title": y_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
310
+ "legend": {"font": {"color": "#ffffff"}},
311
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 80}
312
+ }
313
+ }
314
+
315
+
316
+ def _generate_pie_chart(
317
+ df, x_col: str, y_col: str, title: str,
318
+ group_col: Optional[str], currency_symbol: str,
319
+ colors: List[str], limit: int
320
+ ) -> Dict:
321
+ """Generate pie chart for proportions."""
322
+
323
+ if x_col and x_col in df.columns and y_col and y_col in df.columns:
324
+ aggregated = df.groupby(x_col)[y_col].sum().reset_index()
325
+ aggregated = aggregated.sort_values(y_col, ascending=False).head(8) # Max 8 slices
326
+
327
+ labels = aggregated[x_col].astype(str).tolist()
328
+ values = aggregated[y_col].tolist()
329
+ else:
330
+ labels = []
331
+ values = []
332
+
333
+ return {
334
+ "type": "plotly",
335
+ "chart_type": "pie",
336
+ "data": [{
337
+ "type": "pie",
338
+ "labels": labels,
339
+ "values": values,
340
+ "marker": {"colors": colors[:len(labels)]},
341
+ "textinfo": "label+percent",
342
+ "textposition": "inside",
343
+ "hovertemplate": f"<b>%{{label}}</b><br>{currency_symbol}%{{value:,.0f}}<br>%{{percent}}<extra></extra>"
344
+ }],
345
+ "layout": {
346
+ "title": {"text": title, "font": {"size": 16, "color": "#ffffff"}},
347
+ "paper_bgcolor": "rgba(0,0,0,0)",
348
+ "plot_bgcolor": "rgba(0,0,0,0)",
349
+ "showlegend": True,
350
+ "legend": {"font": {"color": "#ffffff"}},
351
+ "margin": {"l": 30, "r": 30, "t": 50, "b": 30}
352
+ }
353
+ }
354
+
355
+
356
+ def _generate_donut_chart(
357
+ df, x_col: str, y_col: str, title: str,
358
+ group_col: Optional[str], currency_symbol: str,
359
+ colors: List[str], limit: int
360
+ ) -> Dict:
361
+ """Generate donut chart (pie with hole)."""
362
+
363
+ chart = _generate_pie_chart(df, x_col, y_col, title, group_col, currency_symbol, colors, limit)
364
+
365
+ if chart.get("data"):
366
+ chart["data"][0]["hole"] = 0.4
367
+ chart["chart_type"] = "donut"
368
+
369
+ return chart
370
+
371
+
372
+ def _generate_area_chart(
373
+ df, x_col: str, y_col: str, title: str,
374
+ group_col: Optional[str], currency_symbol: str,
375
+ colors: List[str], limit: int
376
+ ) -> Dict:
377
+ """Generate area chart for cumulative trends."""
378
+
379
+ if x_col and x_col in df.columns and y_col and y_col in df.columns:
380
+ aggregated = df.groupby(x_col)[y_col].sum().reset_index()
381
+ aggregated = aggregated.sort_values(x_col).tail(limit)
382
+
383
+ x_values = aggregated[x_col].astype(str).tolist()
384
+ y_values = aggregated[y_col].tolist()
385
+ else:
386
+ x_values = []
387
+ y_values = []
388
+
389
+ return {
390
+ "type": "plotly",
391
+ "chart_type": "area",
392
+ "data": [{
393
+ "type": "scatter",
394
+ "mode": "lines",
395
+ "x": x_values,
396
+ "y": y_values,
397
+ "fill": "tozeroy",
398
+ "fillcolor": f"rgba(249, 115, 22, 0.3)",
399
+ "line": {"color": colors[0], "width": 2},
400
+ "hovertemplate": f"<b>%{{x}}</b><br>{currency_symbol}%{{y:,.0f}}<extra></extra>"
401
+ }],
402
+ "layout": {
403
+ "title": {"text": title, "font": {"size": 16, "color": "#ffffff"}},
404
+ "paper_bgcolor": "rgba(0,0,0,0)",
405
+ "plot_bgcolor": "rgba(0,0,0,0)",
406
+ "xaxis": {"title": x_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
407
+ "yaxis": {"title": y_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
408
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 50}
409
+ }
410
+ }
411
+
412
+
413
+ def _generate_scatter_chart(
414
+ df, x_col: str, y_col: str, title: str,
415
+ group_col: Optional[str], currency_symbol: str,
416
+ colors: List[str], limit: int
417
+ ) -> Dict:
418
+ """Generate scatter plot for correlations."""
419
+
420
+ # For scatter, use first two numeric columns if not specified
421
+ numeric_cols = df.select_dtypes(include=['int64', 'float64']).columns.tolist()
422
+
423
+ if len(numeric_cols) >= 2:
424
+ x_col = x_col if x_col in numeric_cols else numeric_cols[0]
425
+ y_col = y_col if y_col in numeric_cols else numeric_cols[1]
426
+
427
+ x_values = df[x_col].tolist()[:limit*10]
428
+ y_values = df[y_col].tolist()[:limit*10]
429
+ else:
430
+ x_values = []
431
+ y_values = []
432
+
433
+ return {
434
+ "type": "plotly",
435
+ "chart_type": "scatter",
436
+ "data": [{
437
+ "type": "scatter",
438
+ "mode": "markers",
439
+ "x": x_values,
440
+ "y": y_values,
441
+ "marker": {
442
+ "color": colors[0],
443
+ "size": 10,
444
+ "opacity": 0.7,
445
+ "line": {"width": 1, "color": "#ffffff"}
446
+ },
447
+ "hovertemplate": f"<b>{x_col}</b>: %{{x:,.0f}}<br><b>{y_col}</b>: %{{y:,.0f}}<extra></extra>"
448
+ }],
449
+ "layout": {
450
+ "title": {"text": title, "font": {"size": 16, "color": "#ffffff"}},
451
+ "paper_bgcolor": "rgba(0,0,0,0)",
452
+ "plot_bgcolor": "rgba(0,0,0,0)",
453
+ "xaxis": {"title": x_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
454
+ "yaxis": {"title": y_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
455
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 50}
456
+ }
457
+ }
458
+
459
+
460
+ def _generate_forecast_chart(
461
+ df, x_col: str, y_col: str, title: str,
462
+ group_col: Optional[str], currency_symbol: str,
463
+ colors: List[str], limit: int
464
+ ) -> Dict:
465
+ """Generate forecast chart with historical + predicted + confidence bands."""
466
+
467
+ if x_col and x_col in df.columns and y_col and y_col in df.columns:
468
+ aggregated = df.groupby(x_col)[y_col].sum().reset_index()
469
+ aggregated = aggregated.sort_values(x_col)
470
+
471
+ x_values = aggregated[x_col].astype(str).tolist()
472
+ y_values = aggregated[y_col].tolist()
473
+
474
+ # Simple linear forecast (3 periods)
475
+ if len(y_values) >= 2:
476
+ last_val = y_values[-1]
477
+ growth_rate = (y_values[-1] - y_values[0]) / len(y_values) if len(y_values) > 1 else 0
478
+
479
+ forecast_x = [f"Forecast {i+1}" for i in range(3)]
480
+ forecast_y = [last_val + growth_rate * (i + 1) for i in range(3)]
481
+
482
+ # Confidence bands (±15%)
483
+ upper_band = [v * 1.15 for v in forecast_y]
484
+ lower_band = [v * 0.85 for v in forecast_y]
485
+ else:
486
+ forecast_x = []
487
+ forecast_y = []
488
+ upper_band = []
489
+ lower_band = []
490
+ else:
491
+ x_values = []
492
+ y_values = []
493
+ forecast_x = []
494
+ forecast_y = []
495
+ upper_band = []
496
+ lower_band = []
497
+
498
+ traces = [
499
+ # Historical line
500
+ {
501
+ "type": "scatter",
502
+ "mode": "lines+markers",
503
+ "name": "Historical",
504
+ "x": x_values,
505
+ "y": y_values,
506
+ "line": {"color": colors[0], "width": 3},
507
+ "marker": {"size": 8},
508
+ "hovertemplate": f"<b>%{{x}}</b><br>{currency_symbol}%{{y:,.0f}}<extra></extra>"
509
+ },
510
+ # Forecast line
511
+ {
512
+ "type": "scatter",
513
+ "mode": "lines+markers",
514
+ "name": "Forecast",
515
+ "x": forecast_x,
516
+ "y": forecast_y,
517
+ "line": {"color": colors[1], "width": 3, "dash": "dash"},
518
+ "marker": {"size": 8},
519
+ "hovertemplate": f"<b>%{{x}}</b><br>Predicted: {currency_symbol}%{{y:,.0f}}<extra></extra>"
520
+ },
521
+ # Upper confidence band
522
+ {
523
+ "type": "scatter",
524
+ "mode": "lines",
525
+ "name": "Upper Band",
526
+ "x": forecast_x,
527
+ "y": upper_band,
528
+ "line": {"width": 0},
529
+ "showlegend": False,
530
+ "hoverinfo": "skip"
531
+ },
532
+ # Lower confidence band (with fill)
533
+ {
534
+ "type": "scatter",
535
+ "mode": "lines",
536
+ "name": "Lower Band",
537
+ "x": forecast_x,
538
+ "y": lower_band,
539
+ "line": {"width": 0},
540
+ "fill": "tonexty",
541
+ "fillcolor": "rgba(59, 130, 246, 0.2)",
542
+ "showlegend": False,
543
+ "hoverinfo": "skip"
544
+ }
545
+ ]
546
+
547
+ return {
548
+ "type": "plotly",
549
+ "chart_type": "forecast",
550
+ "data": traces,
551
+ "layout": {
552
+ "title": {"text": title, "font": {"size": 16, "color": "#ffffff"}},
553
+ "paper_bgcolor": "rgba(0,0,0,0)",
554
+ "plot_bgcolor": "rgba(0,0,0,0)",
555
+ "xaxis": {"title": "", "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
556
+ "yaxis": {"title": y_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
557
+ "legend": {"font": {"color": "#ffffff"}},
558
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 50}
559
+ }
560
+ }
561
+
562
+
563
+ def _generate_waterfall_chart(
564
+ df, x_col: str, y_col: str, title: str,
565
+ group_col: Optional[str], currency_symbol: str,
566
+ colors: List[str], limit: int
567
+ ) -> Dict:
568
+ """Generate waterfall chart for financial breakdowns."""
569
+
570
+ if x_col and x_col in df.columns and y_col and y_col in df.columns:
571
+ aggregated = df.groupby(x_col)[y_col].sum().reset_index()
572
+ aggregated = aggregated.sort_values(y_col, ascending=False).head(limit)
573
+
574
+ labels = aggregated[x_col].astype(str).tolist()
575
+ values = aggregated[y_col].tolist()
576
+
577
+ # Add total at end
578
+ labels.append("Total")
579
+ values.append(sum(values[:-1]) if len(values) > 1 else values[0] if values else 0)
580
+
581
+ # Waterfall measure types
582
+ measures = ["relative"] * (len(labels) - 1) + ["total"]
583
+ else:
584
+ labels = []
585
+ values = []
586
+ measures = []
587
+
588
+ return {
589
+ "type": "plotly",
590
+ "chart_type": "waterfall",
591
+ "data": [{
592
+ "type": "waterfall",
593
+ "orientation": "v",
594
+ "x": labels,
595
+ "y": values,
596
+ "measure": measures,
597
+ "connector": {"line": {"color": "rgba(255,255,255,0.3)"}},
598
+ "increasing": {"marker": {"color": colors[2] if len(colors) > 2 else "#22c55e"}},
599
+ "decreasing": {"marker": {"color": colors[4] if len(colors) > 4 else "#ef4444"}},
600
+ "totals": {"marker": {"color": colors[0]}},
601
+ "hovertemplate": f"<b>%{{x}}</b><br>{currency_symbol}%{{y:,.0f}}<extra></extra>"
602
+ }],
603
+ "layout": {
604
+ "title": {"text": title, "font": {"size": 16, "color": "#ffffff"}},
605
+ "paper_bgcolor": "rgba(0,0,0,0)",
606
+ "plot_bgcolor": "rgba(0,0,0,0)",
607
+ "xaxis": {"title": "", "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
608
+ "yaxis": {"title": y_col, "gridcolor": "rgba(255,255,255,0.1)", "color": "#ffffff"},
609
+ "margin": {"l": 60, "r": 30, "t": 50, "b": 80}
610
+ }
611
+ }
612
+
613
+
614
+ # ============================================================================
615
+ # MAIN ENTRY POINT
616
+ # ============================================================================
617
+
618
+ def create_chart_from_decision(
619
+ decision, # VisualizationDecision from visualization_intelligence.py
620
+ df,
621
+ currency_symbol: str = "₹",
622
+ user_role: str = "analyst"
623
+ ) -> Optional[Dict]:
624
+ """
625
+ Create a chart payload from a VisualizationDecision.
626
+
627
+ This is the main entry point that bridges the decision layer
628
+ with actual chart generation.
629
+
630
+ IMPORTANT: Only uses REAL data from the DataFrame - no fabrication!
631
+ """
632
+ if not decision.should_render:
633
+ return None
634
+
635
+ # STRICT DATA VALIDATION - Ensure we have real data
636
+ if df is None or df.empty:
637
+ return {"error": "No data available - cannot generate chart"}
638
+
639
+ # Verify columns exist in DataFrame
640
+ if decision.x_column and decision.x_column not in df.columns:
641
+ return {"error": f"Column '{decision.x_column}' not found in uploaded data"}
642
+ if decision.y_column and decision.y_column not in df.columns:
643
+ return {"error": f"Column '{decision.y_column}' not found in uploaded data"}
644
+
645
+ # Determine color scheme based on role
646
+ color_scheme = "default"
647
+ if user_role:
648
+ role_lower = user_role.lower()
649
+ if role_lower == "executive":
650
+ color_scheme = "executive"
651
+ elif role_lower == "finance":
652
+ color_scheme = "finance"
653
+
654
+ chart = generate_dynamic_chart(
655
+ df=df,
656
+ chart_type=decision.chart_type.value,
657
+ x_col=decision.x_column,
658
+ y_col=decision.y_column,
659
+ title=decision.title,
660
+ group_col=decision.group_column,
661
+ currency_symbol=currency_symbol,
662
+ color_scheme=color_scheme,
663
+ limit=decision.limit if hasattr(decision, 'limit') else 10 # Use dynamic limit from query
664
+ )
665
+
666
+ # Add data source metadata to confirm real data usage
667
+ if chart and not chart.get("error"):
668
+ chart["_data_source"] = {
669
+ "type": "uploaded_data",
670
+ "row_count": len(df),
671
+ "columns_used": [c for c in [decision.x_column, decision.y_column, decision.group_column] if c],
672
+ "verified": True
673
+ }
674
+
675
+ return chart
676
+
backend/agents/advanced_report_agent.py ADDED
The diff for this file is too large to render. See raw diff
 
backend/agents/ai_models.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # AI Models Configuration
2
+ """
3
+ Configuration for external AI models available in the chat interface.
4
+ These models are accessed via OpenRouter/LiteLLM.
5
+ """
6
+
7
+ from dataclasses import dataclass
8
+ from typing import Dict, Optional
9
+ from enum import Enum
10
+
11
+
12
+ class AIModelProvider(Enum):
13
+ """AI model providers"""
14
+ GROQ = "groq"
15
+ OPENROUTER = "openrouter"
16
+ TOGETHER = "together"
17
+
18
+
19
+ @dataclass
20
+ class AIModelConfig:
21
+ """Configuration for an AI model"""
22
+ id: str # Frontend mode id (e.g., "deepseek-chat")
23
+ name: str # Display name
24
+ model_path: str # LiteLLM model path
25
+ provider: AIModelProvider
26
+ max_tokens: int
27
+ temperature: float
28
+ description: str
29
+ best_for: str # What this model excels at
30
+ speed: str # "fast", "medium", "slow"
31
+
32
+
33
+ # ============================================================================
34
+ # AI MODEL CONFIGURATIONS - FREE OPENROUTER MODELS ONLY
35
+ # Updated: Dec 2024 - Only verified working models
36
+ # Note: DeepSeek, OpenAI, Gemma removed due to errors
37
+ # ============================================================================
38
+
39
+ AI_MODELS: Dict[str, AIModelConfig] = {
40
+ "mistral-7b": AIModelConfig(
41
+ id="mistral-7b",
42
+ name="Mistral Small",
43
+ model_path="openrouter/mistralai/mistral-small-3.1-24b-instruct:free",
44
+ provider=AIModelProvider.OPENROUTER,
45
+ max_tokens=4000,
46
+ temperature=0.7,
47
+ description="FREE - Fast model for quick decision analysis",
48
+ best_for="Quick analysis, summaries, decisions",
49
+ speed="fast",
50
+ ),
51
+
52
+ "llama-70b": AIModelConfig(
53
+ id="llama-70b",
54
+ name="Llama 3.3 70B",
55
+ model_path="openrouter/meta-llama/llama-3.3-70b-instruct:free",
56
+ provider=AIModelProvider.OPENROUTER,
57
+ max_tokens=4000,
58
+ temperature=0.7,
59
+ description="FREE - Meta's flagship model for comprehensive analysis",
60
+ best_for="Deep analysis, complex reasoning, detailed reports",
61
+ speed="medium",
62
+ ),
63
+ }
64
+
65
+
66
+ def get_ai_model_config(mode_id: str) -> Optional[AIModelConfig]:
67
+ """Get AI model configuration by mode ID"""
68
+ return AI_MODELS.get(mode_id)
69
+
70
+
71
+ def is_ai_model_mode(mode: str) -> bool:
72
+ """Check if the mode is an AI model (not RAG/Graph)"""
73
+ return mode in AI_MODELS
74
+
75
+
76
+ def get_model_path(mode_id: str) -> str:
77
+ """Get LiteLLM model path for a mode"""
78
+ config = get_ai_model_config(mode_id)
79
+ if config:
80
+ return config.model_path
81
+ return None
82
+
83
+
84
+ def get_all_ai_models() -> list:
85
+ """Get list of all AI model configs"""
86
+ return list(AI_MODELS.values())
87
+
88
+
89
+ # ============================================================================
90
+ # AI MODEL SYSTEM PROMPTS
91
+ # ============================================================================
92
+
93
+ AI_MODEL_SYSTEM_PROMPT = """You are an AI Business Analyst. Answer business questions accurately and concisely.
94
+
95
+ RULES:
96
+ 1. Be direct and data-driven
97
+ 2. Use tables for structured data
98
+ 3. Use the user's preferred currency
99
+ 4. If data is not provided, say "I don't have access to your data for this query"
100
+ 5. Never invent numbers or statistics
101
+
102
+ For business queries, provide:
103
+ 1. **One-line answer** (bolded)
104
+ 2. Supporting details or table
105
+ 3. Brief insight if applicable
106
+
107
+ Be professional, confident, and concise."""
108
+
109
+
110
+ def get_ai_model_system_prompt(model_id: str = None) -> str:
111
+ """Get system prompt for AI model mode"""
112
+ return AI_MODEL_SYSTEM_PROMPT
backend/agents/answer_chart_extractor.py ADDED
@@ -0,0 +1,383 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # =============================================================================
2
+ # CHATGPT-STYLE CHART GENERATION - From Answer Data
3
+ # =============================================================================
4
+ #
5
+ # This module extracts chart data directly from LLM answers to ensure
6
+ # charts match the text response exactly - just like real ChatGPT!
7
+ #
8
+
9
+ from typing import Optional, Dict, List, Any, Tuple
10
+ import re
11
+ import json
12
+
13
+
14
+ def extract_chart_data_from_answer(
15
+ answer: str,
16
+ query: str,
17
+ df=None,
18
+ currency_symbol: str = "₹"
19
+ ) -> Optional[Dict]:
20
+ """
21
+ 🧠 CHATGPT-STYLE: Extract chart data from LLM's text answer.
22
+
23
+ Instead of generating charts separately, we extract the data
24
+ that the LLM mentioned in its answer and visualize THAT.
25
+
26
+ Args:
27
+ answer: The LLM's text response
28
+ query: User's original query
29
+ df: Optional DataFrame for validation
30
+ currency_symbol: Currency symbol used
31
+
32
+ Returns:
33
+ Chart specification dict or None
34
+ """
35
+ # Detect what kind of chart is needed
36
+ query_lower = query.lower()
37
+
38
+ # ===========================================================================
39
+ # STEP 1: Detect chart type from query
40
+ # ===========================================================================
41
+ chart_type = "bar" # Default
42
+ if "pie" in query_lower or "distribution" in query_lower or "share" in query_lower:
43
+ chart_type = "pie"
44
+ elif "line" in query_lower or "trend" in query_lower or "over time" in query_lower:
45
+ chart_type = "line"
46
+ elif "bar" in query_lower or "top" in query_lower or "ranking" in query_lower:
47
+ chart_type = "bar"
48
+
49
+ # ===========================================================================
50
+ # STEP 2: Extract structured data from answer
51
+ # ===========================================================================
52
+
53
+ # Pattern 1: Markdown table extraction
54
+ table_data = _extract_table_from_markdown(answer)
55
+ if table_data:
56
+ return _create_chart_from_table(table_data, chart_type, query, currency_symbol)
57
+
58
+ # Pattern 2: Numbered list with values (e.g., "1. Glovo Ltd - $1,153,500")
59
+ list_data = _extract_list_from_answer(answer)
60
+ if list_data:
61
+ return _create_chart_from_list(list_data, chart_type, query, currency_symbol)
62
+
63
+ # Pattern 3: Inline mentions (e.g., "Your top customer is Glovo Ltd with $1.15M")
64
+ inline_data = _extract_inline_data(answer, currency_symbol)
65
+ if inline_data:
66
+ return _create_chart_from_list(inline_data, chart_type, query, currency_symbol)
67
+
68
+ return None
69
+
70
+
71
+ def _extract_table_from_markdown(answer: str) -> Optional[List[Dict]]:
72
+ """
73
+ Extract data from markdown tables in the answer.
74
+
75
+ Properly parses tables like:
76
+ | Customer | Revenue |
77
+ |----------|---------|
78
+ | Glovo Ltd | $1,153,500 |
79
+ """
80
+ lines = answer.split('\n')
81
+
82
+ # Find table start (header row)
83
+ table_start = -1
84
+ for i, line in enumerate(lines):
85
+ # Table header has at least 2 pipe characters and no separator
86
+ if line.count('|') >= 2 and '---' not in line and '===' not in line:
87
+ # Check if next line is separator
88
+ if i + 1 < len(lines) and ('---' in lines[i + 1] or '===' in lines[i + 1]):
89
+ table_start = i
90
+ break
91
+
92
+ if table_start < 0:
93
+ return None
94
+
95
+ # Parse header
96
+ header_line = lines[table_start]
97
+ headers = []
98
+ for cell in header_line.split('|'):
99
+ cell = cell.strip()
100
+ # Clean markdown formatting
101
+ cell = re.sub(r'\*\*([^*]+)\*\*', r'\1', cell) # Remove bold
102
+ cell = re.sub(r'\*([^*]+)\*', r'\1', cell) # Remove italic
103
+ if cell and cell not in ['', '-', '---']:
104
+ headers.append(cell)
105
+
106
+ if len(headers) < 2:
107
+ return None
108
+
109
+ # Parse data rows (skip header and separator)
110
+ rows = []
111
+ for i in range(table_start + 2, len(lines)):
112
+ line = lines[i].strip()
113
+
114
+ # Stop at empty line or non-table line
115
+ if not line or '|' not in line:
116
+ break
117
+
118
+ # Parse cells
119
+ cells = []
120
+ for cell in line.split('|'):
121
+ cell = cell.strip()
122
+ # Clean markdown formatting
123
+ cell = re.sub(r'\*\*([^*]+)\*\*', r'\1', cell) # Remove bold
124
+ cell = re.sub(r'\*([^*]+)\*', r'\1', cell) # Remove italic
125
+ cell = cell.replace('\u202f', '') # Remove narrow non-breaking space
126
+ if cell and cell not in ['', '-', '---']:
127
+ cells.append(cell)
128
+
129
+ if len(cells) >= 2:
130
+ row_dict = {}
131
+ for j, header in enumerate(headers):
132
+ if j < len(cells):
133
+ row_dict[header] = cells[j]
134
+ if row_dict:
135
+ rows.append(row_dict)
136
+
137
+ # Validate we have actual data, not markdown artifacts
138
+ if rows:
139
+ # Check first row doesn't contain markdown syntax
140
+ first_row_values = list(rows[0].values())
141
+ for val in first_row_values:
142
+ if '|' in str(val) or '---' in str(val):
143
+ return None # Still has markdown artifacts
144
+
145
+ return rows if rows else None
146
+
147
+
148
+
149
+ def _extract_list_from_answer(answer: str) -> Optional[List[Dict]]:
150
+ """Extract data from numbered or bulleted lists."""
151
+ # Pattern: "1. Name - $1,234,567" or "• Name: $1,234,567"
152
+ patterns = [
153
+ r'(?:\d+\.|•|-)\s*([^:\-–—]+?)[\s:\-–—]+[₹$€£]?([\d,]+(?:\.\d+)?)',
154
+ r'(?:\d+\.|•|-)\s*\*\*([^*]+)\*\*.*?[₹$€£]?([\d,]+(?:\.\d+)?)',
155
+ ]
156
+
157
+ items = []
158
+ for pattern in patterns:
159
+ matches = re.findall(pattern, answer)
160
+ for match in matches:
161
+ name = match[0].strip()
162
+ value_str = match[1].replace(',', '')
163
+ try:
164
+ value = float(value_str)
165
+ if value > 0:
166
+ items.append({'name': name, 'value': value})
167
+ except ValueError:
168
+ continue
169
+
170
+ # Deduplicate by name
171
+ seen = set()
172
+ unique_items = []
173
+ for item in items:
174
+ if item['name'] not in seen:
175
+ seen.add(item['name'])
176
+ unique_items.append(item)
177
+
178
+ return unique_items[:10] if unique_items else None
179
+
180
+
181
+ def _extract_inline_data(answer: str, currency_symbol: str = "₹") -> Optional[List[Dict]]:
182
+ """Extract entity-value pairs mentioned inline in the answer."""
183
+ # Pattern: "Customer_Name with $1,234,567" or "Customer_Name ($1,234,567)"
184
+ patterns = [
185
+ r'(\b[A-Z][a-zA-Z0-9_\s]+?)\s+(?:with|at|generated|earned)\s+[₹$€£]?([\d,]+(?:\.\d+)?)',
186
+ r'(\b[A-Z][a-zA-Z0-9_\s]+?)\s*\([₹$€£]?([\d,]+(?:\.\d+)?)\)',
187
+ r'\*\*([^*]+?)\*\*.*?[₹$€£]?([\d,]+(?:\.\d+)?)',
188
+ ]
189
+
190
+ items = []
191
+ for pattern in patterns:
192
+ matches = re.findall(pattern, answer)
193
+ for match in matches:
194
+ name = match[0].strip()
195
+ # Skip common phrases
196
+ if name.lower() in ['your', 'the', 'total', 'revenue', 'amount', 'average']:
197
+ continue
198
+ value_str = match[1].replace(',', '')
199
+ try:
200
+ value = float(value_str)
201
+ if value > 0 and len(name) > 2:
202
+ items.append({'name': name, 'value': value})
203
+ except ValueError:
204
+ continue
205
+
206
+ return items[:10] if items else None
207
+
208
+
209
+ def _create_chart_from_table(
210
+ table_data: List[Dict],
211
+ chart_type: str,
212
+ query: str,
213
+ currency_symbol: str
214
+ ) -> Dict:
215
+ """Create chart specification from table data."""
216
+ if not table_data:
217
+ return None
218
+
219
+ # Find the name column (first text column)
220
+ name_col = None
221
+ value_col = None
222
+
223
+ first_row = table_data[0]
224
+ for key, val in first_row.items():
225
+ # Detect name column
226
+ if name_col is None and not _is_numeric(val):
227
+ name_col = key
228
+ # Detect value column (contains numbers)
229
+ elif _is_numeric(val):
230
+ value_col = key
231
+
232
+ if not name_col or not value_col:
233
+ return None
234
+
235
+ # Extract names and values
236
+ names = []
237
+ values = []
238
+ for row in table_data:
239
+ name = row.get(name_col, '')
240
+ value_str = row.get(value_col, '0')
241
+ try:
242
+ # Clean value string
243
+ value_clean = re.sub(r'[^\d.]', '', str(value_str))
244
+ value = float(value_clean) if value_clean else 0
245
+ names.append(name)
246
+ values.append(value)
247
+ except ValueError:
248
+ continue
249
+
250
+ return _build_plotly_chart(names, values, chart_type, query, currency_symbol)
251
+
252
+
253
+ def _create_chart_from_list(
254
+ list_data: List[Dict],
255
+ chart_type: str,
256
+ query: str,
257
+ currency_symbol: str
258
+ ) -> Dict:
259
+ """Create chart specification from list data."""
260
+ if not list_data:
261
+ return None
262
+
263
+ names = [item['name'] for item in list_data]
264
+ values = [item['value'] for item in list_data]
265
+
266
+ return _build_plotly_chart(names, values, chart_type, query, currency_symbol)
267
+
268
+
269
+ def _build_plotly_chart(
270
+ names: List[str],
271
+ values: List[float],
272
+ chart_type: str,
273
+ query: str,
274
+ currency_symbol: str
275
+ ) -> Dict:
276
+ """Build a Plotly chart specification."""
277
+ # Generate title from query
278
+ title = _generate_chart_title(query, names)
279
+
280
+ # Format values for display
281
+ def format_value(v):
282
+ if v >= 1_000_000:
283
+ return f"{currency_symbol}{v/1_000_000:.1f}M"
284
+ elif v >= 1_000:
285
+ return f"{currency_symbol}{v/1_000:.1f}K"
286
+ else:
287
+ return f"{currency_symbol}{v:,.0f}"
288
+
289
+ if chart_type == "pie":
290
+ return {
291
+ "data": [{
292
+ "type": "pie",
293
+ "labels": names,
294
+ "values": values,
295
+ "textinfo": "label+percent",
296
+ "hovertemplate": "%{label}<br>" + currency_symbol + "%{value:,.0f}<br>%{percent}<extra></extra>",
297
+ "marker": {
298
+ "colors": ['#FF6B35', '#4ECDC4', '#45B7D1', '#96CEB4', '#FFEAA7',
299
+ '#DDA0DD', '#98D8C8', '#F7DC6F', '#BB8FCE', '#85C1E9'][:len(names)]
300
+ }
301
+ }],
302
+ "layout": {
303
+ "title": {"text": title, "font": {"size": 16}},
304
+ "showlegend": True,
305
+ "height": 400
306
+ }
307
+ }
308
+ elif chart_type == "line":
309
+ return {
310
+ "data": [{
311
+ "type": "scatter",
312
+ "mode": "lines+markers",
313
+ "x": names,
314
+ "y": values,
315
+ "line": {"color": "#FF6B35", "width": 3},
316
+ "marker": {"size": 8}
317
+ }],
318
+ "layout": {
319
+ "title": {"text": title, "font": {"size": 16}},
320
+ "xaxis": {"title": ""},
321
+ "yaxis": {"title": "Amount", "tickprefix": currency_symbol},
322
+ "height": 400
323
+ }
324
+ }
325
+ else: # bar chart (default)
326
+ return {
327
+ "data": [{
328
+ "type": "bar",
329
+ "x": names,
330
+ "y": values,
331
+ "text": [format_value(v) for v in values],
332
+ "textposition": "auto",
333
+ "marker": {
334
+ "color": ['#FF6B35', '#4ECDC4', '#45B7D1', '#96CEB4', '#FFEAA7',
335
+ '#DDA0DD', '#98D8C8', '#F7DC6F', '#BB8FCE', '#85C1E9'][:len(names)]
336
+ }
337
+ }],
338
+ "layout": {
339
+ "title": {"text": title, "font": {"size": 16}},
340
+ "xaxis": {"title": "", "tickangle": -45},
341
+ "yaxis": {"title": "Amount", "tickprefix": currency_symbol},
342
+ "height": 400,
343
+ "margin": {"b": 100}
344
+ }
345
+ }
346
+
347
+
348
+ def _generate_chart_title(query: str, names: List[str]) -> str:
349
+ """Generate a chart title from the query."""
350
+ query_lower = query.lower()
351
+
352
+ # Extract entity type from query
353
+ entity = "Items"
354
+ if "customer" in query_lower:
355
+ entity = "Customers"
356
+ elif "product" in query_lower:
357
+ entity = "Products"
358
+ elif "region" in query_lower:
359
+ entity = "Regions"
360
+ elif "category" in query_lower:
361
+ entity = "Categories"
362
+
363
+ # Detect top vs bottom
364
+ if "bottom" in query_lower or "low" in query_lower or "worst" in query_lower:
365
+ return f"📉 Bottom {len(names)} {entity} by Revenue"
366
+ elif "top" in query_lower or "best" in query_lower or "highest" in query_lower:
367
+ return f"🏆 Top {len(names)} {entity} by Revenue"
368
+ elif "vs" in query_lower or "compare" in query_lower:
369
+ return f"📊 {entity} Comparison"
370
+ else:
371
+ return f"📊 {entity} by Revenue"
372
+
373
+
374
+ def _is_numeric(value: str) -> bool:
375
+ """Check if a value looks numeric."""
376
+ if not value:
377
+ return False
378
+ cleaned = re.sub(r'[₹$€£,%\s]', '', str(value))
379
+ try:
380
+ float(cleaned)
381
+ return True
382
+ except ValueError:
383
+ return False
backend/agents/autonomous_charts.py ADDED
@@ -0,0 +1,400 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ AUTONOMOUS CHART GENERATOR - 100% LLM-Driven from User Data
3
+ =============================================================
4
+
5
+ Generates charts, mindmaps, and visualizations AUTONOMOUSLY
6
+ based on user's query and their uploaded data.
7
+
8
+ NO HARDCODING - Everything is determined by LLM at runtime:
9
+ - Chart type selection
10
+ - Data extraction
11
+ - Colors and styling
12
+ - Labels and titles
13
+
14
+ Supports:
15
+ - Bar, Line, Pie, Scatter charts
16
+ - Mindmaps (hierarchical tree diagrams)
17
+ - Network graphs (relationships)
18
+ - Heatmaps
19
+ - Tables
20
+ """
21
+
22
+ import json
23
+ import logging
24
+ from typing import Dict, List, Optional, Any, Tuple
25
+ import pandas as pd
26
+
27
+ from core.llm import chat
28
+
29
+ logger = logging.getLogger(__name__)
30
+
31
+
32
+ def autonomous_chart_decision(query: str, columns: List[str], sample_data: str) -> Dict:
33
+ """
34
+ Use LLM to decide what chart to create - NO HARDCODING!
35
+ """
36
+ try:
37
+ prompt = f"""Analyze this query and data to determine the best visualization.
38
+
39
+ USER QUERY: "{query}"
40
+ DATA COLUMNS: {columns}
41
+ SAMPLE DATA: {sample_data[:500]}
42
+
43
+ Decide:
44
+ 1. Should a chart be generated? (yes/no)
45
+ 2. What chart type? (bar, line, pie, scatter, heatmap, table, mindmap, network)
46
+ 3. Which column for X-axis/categories?
47
+ 4. Which column for Y-axis/values?
48
+ 5. Chart title?
49
+
50
+ Return JSON:
51
+ {{
52
+ "generate_chart": true/false,
53
+ "chart_type": "bar/line/pie/scatter/heatmap/table/mindmap/network",
54
+ "x_column": "column_name",
55
+ "y_column": "column_name",
56
+ "title": "Chart Title",
57
+ "reason": "why this chart type"
58
+ }}
59
+
60
+ JSON:"""
61
+
62
+ result = chat(prompt, temperature=0.1, max_tokens=200)
63
+
64
+ # Parse JSON
65
+ result = result.strip()
66
+ if '```' in result:
67
+ result = result.split('```')[1]
68
+ if result.startswith('json'):
69
+ result = result[4:]
70
+
71
+ start = result.find('{')
72
+ end = result.rfind('}') + 1
73
+ if start >= 0 and end > start:
74
+ result = result[start:end]
75
+
76
+ decision = json.loads(result)
77
+ return decision
78
+
79
+ except Exception as e:
80
+ logger.warning(f"Chart decision error: {e}")
81
+ return {"generate_chart": False, "reason": str(e)}
82
+
83
+
84
+ def autonomous_data_extraction(df: pd.DataFrame, query: str, chart_type: str) -> Dict:
85
+ """
86
+ Autonomously extract the right data for visualization.
87
+ """
88
+ try:
89
+ columns = list(df.columns)
90
+ sample = df.head(5).to_string()
91
+
92
+ prompt = f"""Extract data for a {chart_type} chart from this DataFrame.
93
+
94
+ QUERY: "{query}"
95
+ COLUMNS: {columns}
96
+ SAMPLE DATA:
97
+ {sample}
98
+
99
+ TASK: Return JSON with the exact column names to use:
100
+ {{
101
+ "x_column": "exact_column_name_for_x",
102
+ "y_column": "exact_column_name_for_y",
103
+ "group_by": "column_to_group_by or null",
104
+ "aggregate": "sum/mean/count/max/min",
105
+ "top_n": 10,
106
+ "sort_by": "asc/desc"
107
+ }}
108
+
109
+ Use EXACT column names from the list above.
110
+ JSON:"""
111
+
112
+ result = chat(prompt, temperature=0.1, max_tokens=150)
113
+
114
+ # Parse
115
+ result = result.strip()
116
+ if '```' in result:
117
+ result = result.split('```')[1]
118
+ if result.startswith('json'):
119
+ result = result[4:]
120
+
121
+ start = result.find('{')
122
+ end = result.rfind('}') + 1
123
+ if start >= 0 and end > start:
124
+ result = result[start:end]
125
+
126
+ return json.loads(result)
127
+
128
+ except Exception as e:
129
+ logger.warning(f"Data extraction error: {e}")
130
+ # Fallback: use first string and numeric columns
131
+ str_cols = [c for c in df.columns if df[c].dtype == 'object']
132
+ num_cols = [c for c in df.columns if df[c].dtype in ['int64', 'float64']]
133
+ return {
134
+ "x_column": str_cols[0] if str_cols else columns[0],
135
+ "y_column": num_cols[0] if num_cols else columns[-1],
136
+ "aggregate": "sum",
137
+ "top_n": 10
138
+ }
139
+
140
+
141
+ def generate_autonomous_chart(
142
+ query: str,
143
+ df: pd.DataFrame,
144
+ chart_type: str = None,
145
+ x_column: str = None,
146
+ y_column: str = None
147
+ ) -> Optional[str]:
148
+ """
149
+ Generate chart autonomously from user data.
150
+ Returns Plotly chart as JSON or markdown code block.
151
+ """
152
+ if df is None or df.empty:
153
+ return None
154
+
155
+ try:
156
+ columns = list(df.columns)
157
+ sample = df.head(3).to_string()
158
+
159
+ # Step 1: Get chart decision if not specified
160
+ if not chart_type:
161
+ decision = autonomous_chart_decision(query, columns, sample)
162
+ if not decision.get("generate_chart", False):
163
+ return None
164
+ chart_type = decision.get("chart_type", "bar")
165
+ x_column = decision.get("x_column")
166
+ y_column = decision.get("y_column")
167
+
168
+ # Step 2: Extract data
169
+ extraction = autonomous_data_extraction(df, query, chart_type)
170
+ x_col = x_column or extraction.get("x_column")
171
+ y_col = y_column or extraction.get("y_column")
172
+ aggregate = extraction.get("aggregate", "sum")
173
+ top_n = extraction.get("top_n", 10)
174
+
175
+ # Validate columns exist
176
+ if x_col not in df.columns:
177
+ x_col = columns[0]
178
+ if y_col not in df.columns:
179
+ y_col = columns[-1] if len(columns) > 1 else columns[0]
180
+
181
+ # Step 3: Prepare data
182
+ if aggregate == "count":
183
+ chart_data = df[x_col].value_counts().head(top_n)
184
+ labels = chart_data.index.tolist()
185
+ values = chart_data.values.tolist()
186
+ else:
187
+ # Group and aggregate
188
+ try:
189
+ if df[y_col].dtype in ['int64', 'float64']:
190
+ grouped = df.groupby(x_col)[y_col].agg(aggregate).head(top_n)
191
+ labels = grouped.index.tolist()
192
+ values = grouped.values.tolist()
193
+ else:
194
+ # Count if not numeric
195
+ chart_data = df[x_col].value_counts().head(top_n)
196
+ labels = chart_data.index.tolist()
197
+ values = chart_data.values.tolist()
198
+ except:
199
+ chart_data = df[x_col].value_counts().head(top_n)
200
+ labels = chart_data.index.tolist()
201
+ values = chart_data.values.tolist()
202
+
203
+ # Step 4: Build Plotly chart
204
+ title = f"{y_col} by {x_col}" if y_col != x_col else f"Distribution of {x_col}"
205
+
206
+ if chart_type == "pie":
207
+ chart_json = {
208
+ "data": [{
209
+ "type": "pie",
210
+ "labels": [str(l) for l in labels],
211
+ "values": values,
212
+ "hole": 0.4,
213
+ "marker": {"colors": ["#14b8a6", "#6366f1", "#f59e0b", "#ef4444", "#8b5cf6", "#06b6d4", "#84cc16", "#f97316"]}
214
+ }],
215
+ "layout": {
216
+ "title": {"text": title, "font": {"color": "#fff"}},
217
+ "paper_bgcolor": "rgba(0,0,0,0)",
218
+ "plot_bgcolor": "rgba(0,0,0,0)",
219
+ "font": {"color": "#9ca3af"},
220
+ "showlegend": True
221
+ }
222
+ }
223
+ elif chart_type == "line":
224
+ chart_json = {
225
+ "data": [{
226
+ "type": "scatter",
227
+ "mode": "lines+markers",
228
+ "x": [str(l) for l in labels],
229
+ "y": values,
230
+ "line": {"color": "#14b8a6", "width": 3},
231
+ "marker": {"size": 8, "color": "#14b8a6"}
232
+ }],
233
+ "layout": {
234
+ "title": {"text": title, "font": {"color": "#fff"}},
235
+ "paper_bgcolor": "rgba(0,0,0,0)",
236
+ "plot_bgcolor": "rgba(0,0,0,0)",
237
+ "font": {"color": "#9ca3af"},
238
+ "xaxis": {"gridcolor": "#374151"},
239
+ "yaxis": {"gridcolor": "#374151"}
240
+ }
241
+ }
242
+ elif chart_type == "scatter":
243
+ chart_json = {
244
+ "data": [{
245
+ "type": "scatter",
246
+ "mode": "markers",
247
+ "x": [str(l) for l in labels],
248
+ "y": values,
249
+ "marker": {"size": 12, "color": "#6366f1", "opacity": 0.7}
250
+ }],
251
+ "layout": {
252
+ "title": {"text": title, "font": {"color": "#fff"}},
253
+ "paper_bgcolor": "rgba(0,0,0,0)",
254
+ "plot_bgcolor": "rgba(0,0,0,0)",
255
+ "font": {"color": "#9ca3af"}
256
+ }
257
+ }
258
+ else: # bar (default)
259
+ chart_json = {
260
+ "data": [{
261
+ "type": "bar",
262
+ "x": [str(l) for l in labels],
263
+ "y": values,
264
+ "marker": {
265
+ "color": values,
266
+ "colorscale": [[0, "#14b8a6"], [1, "#6366f1"]]
267
+ }
268
+ }],
269
+ "layout": {
270
+ "title": {"text": title, "font": {"color": "#fff"}},
271
+ "paper_bgcolor": "rgba(0,0,0,0)",
272
+ "plot_bgcolor": "rgba(0,0,0,0)",
273
+ "font": {"color": "#9ca3af"},
274
+ "xaxis": {"gridcolor": "#374151"},
275
+ "yaxis": {"gridcolor": "#374151"}
276
+ }
277
+ }
278
+
279
+ return f"\n\n```plotly_chart\n{json.dumps(chart_json)}\n```"
280
+
281
+ except Exception as e:
282
+ logger.error(f"Autonomous chart error: {e}")
283
+ return None
284
+
285
+
286
+ def generate_mindmap(query: str, df: pd.DataFrame) -> Optional[str]:
287
+ """
288
+ Generate mindmap visualization from user data.
289
+ """
290
+ if df is None or df.empty:
291
+ return None
292
+
293
+ try:
294
+ columns = list(df.columns)
295
+ sample = df.head(10).to_string()
296
+
297
+ # Use LLM to structure the mindmap
298
+ prompt = f"""Create a mindmap structure from this data.
299
+
300
+ QUERY: "{query}"
301
+ COLUMNS: {columns}
302
+ DATA:
303
+ {sample}
304
+
305
+ Create a hierarchical mindmap with:
306
+ - Central topic
307
+ - Main branches (3-5)
308
+ - Sub-items under each branch
309
+
310
+ Return JSON:
311
+ {{
312
+ "center": "main topic",
313
+ "branches": [
314
+ {{"name": "Branch 1", "items": ["item1", "item2"]}},
315
+ {{"name": "Branch 2", "items": ["item1", "item2"]}}
316
+ ]
317
+ }}
318
+
319
+ Use REAL data from above.
320
+ JSON:"""
321
+
322
+ result = chat(prompt, temperature=0.3, max_tokens=500)
323
+
324
+ # Parse
325
+ result = result.strip()
326
+ if '```' in result:
327
+ result = result.split('```')[1]
328
+ if result.startswith('json'):
329
+ result = result[4:]
330
+
331
+ start = result.find('{')
332
+ end = result.rfind('}') + 1
333
+ if start >= 0 and end > start:
334
+ result = result[start:end]
335
+
336
+ mindmap = json.loads(result)
337
+
338
+ # Build Plotly treemap/sunburst
339
+ labels = [mindmap["center"]]
340
+ parents = [""]
341
+ values = [10]
342
+
343
+ for branch in mindmap.get("branches", []):
344
+ branch_name = branch["name"]
345
+ labels.append(branch_name)
346
+ parents.append(mindmap["center"])
347
+ values.append(5)
348
+
349
+ for item in branch.get("items", [])[:5]:
350
+ labels.append(str(item))
351
+ parents.append(branch_name)
352
+ values.append(1)
353
+
354
+ chart_json = {
355
+ "data": [{
356
+ "type": "sunburst",
357
+ "labels": labels,
358
+ "parents": parents,
359
+ "values": values,
360
+ "branchvalues": "total",
361
+ "marker": {"colorscale": "Teal"}
362
+ }],
363
+ "layout": {
364
+ "title": {"text": f"Mindmap: {mindmap['center']}", "font": {"color": "#fff"}},
365
+ "paper_bgcolor": "rgba(0,0,0,0)",
366
+ "font": {"color": "#9ca3af"}
367
+ }
368
+ }
369
+
370
+ return f"\n\n```plotly_chart\n{json.dumps(chart_json)}\n```"
371
+
372
+ except Exception as e:
373
+ logger.error(f"Mindmap error: {e}")
374
+ return None
375
+
376
+
377
+ def should_generate_visualization(query: str) -> Tuple[bool, str]:
378
+ """
379
+ Determine if we should generate a visualization and what type.
380
+ """
381
+ query_lower = query.lower()
382
+
383
+ # Visualization keywords
384
+ if any(w in query_lower for w in ['chart', 'graph', 'visualize', 'plot', 'show me']):
385
+ if 'pie' in query_lower:
386
+ return True, 'pie'
387
+ elif 'line' in query_lower or 'trend' in query_lower:
388
+ return True, 'line'
389
+ elif 'scatter' in query_lower:
390
+ return True, 'scatter'
391
+ else:
392
+ return True, 'bar'
393
+
394
+ if 'mindmap' in query_lower or 'mind map' in query_lower:
395
+ return True, 'mindmap'
396
+
397
+ if any(w in query_lower for w in ['top', 'bottom', 'highest', 'lowest', 'compare', 'breakdown', 'distribution']):
398
+ return True, 'bar'
399
+
400
+ return False, 'none'
backend/agents/autopilot.py ADDED
@@ -0,0 +1,830 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🧠 AGENTIC AUTOPILOT ENGINE v1.0
3
+ =================================
4
+
5
+ The world's first fully autonomous data science agent.
6
+
7
+ Given a dataset and a goal (or no goal at all), this engine:
8
+ 1. Profiles the data autonomously
9
+ 2. Cleans and prepares it
10
+ 3. Discovers patterns and insights
11
+ 4. Trains the best ML model
12
+ 5. Generates visualizations
13
+ 6. Produces a complete analysis report
14
+ 7. Deploys a prediction API
15
+
16
+ All with ZERO human intervention. The user just uploads data and watches.
17
+
18
+ Architecture:
19
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
20
+ │ PLANNER │────▶│ EXECUTOR │────▶│ EVALUATOR │
21
+ │ (Goal → Plan)│ │ (Run Steps) │ │ (Check/Fix) │
22
+ └──────────────┘ └──────────────┘ └──────────────┘
23
+ ▲ │
24
+ └─────────────────────────────────────────┘
25
+ (Re-plan if needed)
26
+ """
27
+
28
+ import logging
29
+ import json
30
+ import time
31
+ import traceback
32
+ import pandas as pd
33
+ import numpy as np
34
+ from typing import Dict, Any, List, Optional, Callable
35
+ from dataclasses import dataclass, field
36
+ from datetime import datetime
37
+ from enum import Enum
38
+ from pathlib import Path
39
+ import asyncio
40
+
41
+ logger = logging.getLogger(__name__)
42
+
43
+
44
+ class StepStatus(Enum):
45
+ PENDING = "pending"
46
+ RUNNING = "running"
47
+ COMPLETED = "completed"
48
+ FAILED = "failed"
49
+ SKIPPED = "skipped"
50
+
51
+
52
+ class AutopilotPhase(Enum):
53
+ PLANNING = "planning"
54
+ DATA_PROFILING = "data_profiling"
55
+ DATA_CLEANING = "data_cleaning"
56
+ FEATURE_DISCOVERY = "feature_discovery"
57
+ INSIGHT_EXTRACTION = "insight_extraction"
58
+ VISUALIZATION = "visualization"
59
+ MODEL_TRAINING = "model_training"
60
+ MODEL_EVALUATION = "model_evaluation"
61
+ CODE_GENERATION = "code_generation"
62
+ REPORT_GENERATION = "report_generation"
63
+ COMPLETE = "complete"
64
+
65
+
66
+ @dataclass
67
+ class AutopilotStep:
68
+ """A single step in the autopilot execution plan."""
69
+ id: str
70
+ phase: AutopilotPhase
71
+ title: str
72
+ description: str
73
+ status: StepStatus = StepStatus.PENDING
74
+ result: Optional[Dict[str, Any]] = None
75
+ error: Optional[str] = None
76
+ started_at: Optional[float] = None
77
+ completed_at: Optional[float] = None
78
+ progress: float = 0.0 # 0-100
79
+
80
+ def to_dict(self):
81
+ return {
82
+ "id": self.id,
83
+ "phase": self.phase.value,
84
+ "title": self.title,
85
+ "description": self.description,
86
+ "status": self.status.value,
87
+ "result": self.result,
88
+ "error": self.error,
89
+ "duration_ms": int((self.completed_at - self.started_at) * 1000) if self.started_at and self.completed_at else None,
90
+ "progress": self.progress
91
+ }
92
+
93
+
94
+ @dataclass
95
+ class AutopilotState:
96
+ """Full state of an autopilot run."""
97
+ user_id: str
98
+ session_id: str
99
+ goal: str
100
+ filename: str
101
+ df: Optional[pd.DataFrame] = None
102
+ target_column: Optional[str] = None
103
+ task_type: Optional[str] = None
104
+ steps: List[AutopilotStep] = field(default_factory=list)
105
+ current_step_index: int = 0
106
+ insights: List[str] = field(default_factory=list)
107
+ charts: List[Dict[str, Any]] = field(default_factory=list)
108
+ model_result: Optional[Dict[str, Any]] = None
109
+ report: Optional[str] = None
110
+ started_at: float = 0.0
111
+ completed_at: Optional[float] = None
112
+ cancelled: bool = False
113
+
114
+ def to_dict(self):
115
+ return {
116
+ "session_id": self.session_id,
117
+ "goal": self.goal,
118
+ "filename": self.filename,
119
+ "target_column": self.target_column,
120
+ "task_type": self.task_type,
121
+ "current_step": self.current_step_index,
122
+ "total_steps": len(self.steps),
123
+ "steps": [s.to_dict() for s in self.steps],
124
+ "insights": self.insights,
125
+ "charts": self.charts,
126
+ "elapsed_seconds": round(time.time() - self.started_at, 1) if self.started_at else 0,
127
+ }
128
+
129
+
130
+ class AgenticAutopilot:
131
+ """
132
+ The Agentic Autopilot — an autonomous data science agent.
133
+
134
+ Usage:
135
+ autopilot = AgenticAutopilot(user_id, df, goal="Predict customer churn")
136
+ async for event in autopilot.run():
137
+ # event is a dict: {"type": "step_start"|"step_complete"|"insight"|"chart"|...}
138
+ yield event
139
+ """
140
+
141
+ def __init__(self, user_id: str, df: pd.DataFrame, filename: str = "data.csv",
142
+ goal: str = "", target_column: str = None, session_id: str = None):
143
+ import uuid
144
+ self.state = AutopilotState(
145
+ user_id=user_id,
146
+ session_id=session_id or str(uuid.uuid4()),
147
+ goal=goal or "Perform comprehensive autonomous data analysis",
148
+ filename=filename,
149
+ df=df,
150
+ target_column=target_column,
151
+ started_at=time.time()
152
+ )
153
+ self._event_callback: Optional[Callable] = None
154
+ self._build_plan()
155
+
156
+ def _build_plan(self):
157
+ """Build the execution plan based on the goal and data."""
158
+ steps = [
159
+ AutopilotStep(
160
+ id="plan", phase=AutopilotPhase.PLANNING,
161
+ title="🧠 Analyzing Goal & Planning Strategy",
162
+ description="Understanding your data and formulating an optimal analysis plan"
163
+ ),
164
+ AutopilotStep(
165
+ id="profile", phase=AutopilotPhase.DATA_PROFILING,
166
+ title="🔍 Deep Data Profiling",
167
+ description="Scanning every column — types, distributions, missing values, correlations"
168
+ ),
169
+ AutopilotStep(
170
+ id="clean", phase=AutopilotPhase.DATA_CLEANING,
171
+ title="🧹 Autonomous Data Cleaning",
172
+ description="Fixing missing values, outliers, duplicates, and type errors"
173
+ ),
174
+ AutopilotStep(
175
+ id="features", phase=AutopilotPhase.FEATURE_DISCOVERY,
176
+ title="⚡ Intelligent Feature Discovery",
177
+ description="Engineering new features and selecting the most predictive ones"
178
+ ),
179
+ AutopilotStep(
180
+ id="insights", phase=AutopilotPhase.INSIGHT_EXTRACTION,
181
+ title="💡 Extracting Key Insights",
182
+ description="Finding patterns, trends, anomalies, and correlations in your data"
183
+ ),
184
+ AutopilotStep(
185
+ id="viz", phase=AutopilotPhase.VISUALIZATION,
186
+ title="📊 Generating Visualizations",
187
+ description="Creating the most informative charts and plots for your data"
188
+ ),
189
+ AutopilotStep(
190
+ id="train", phase=AutopilotPhase.MODEL_TRAINING,
191
+ title="🏋️ Training ML Models",
192
+ description="Testing multiple algorithms with cross-validation to find the best model"
193
+ ),
194
+ AutopilotStep(
195
+ id="eval", phase=AutopilotPhase.MODEL_EVALUATION,
196
+ title="📈 Evaluating & Explaining Model",
197
+ description="Computing metrics, feature importance, and reliability scores"
198
+ ),
199
+ AutopilotStep(
200
+ id="code", phase=AutopilotPhase.CODE_GENERATION,
201
+ title="💻 Generating Deployment Code",
202
+ description="Creating a production-ready prediction API you can run locally"
203
+ ),
204
+ AutopilotStep(
205
+ id="report", phase=AutopilotPhase.REPORT_GENERATION,
206
+ title="📋 Compiling Analysis Report",
207
+ description="Summarizing everything into an executive-ready report"
208
+ ),
209
+ ]
210
+ self.state.steps = steps
211
+
212
+ async def run(self):
213
+ """
214
+ Execute the full autopilot pipeline.
215
+ Yields SSE-compatible event dicts as it progresses.
216
+ """
217
+ yield self._event("session_start", {
218
+ "session_id": self.state.session_id,
219
+ "goal": self.state.goal,
220
+ "total_steps": len(self.state.steps),
221
+ "filename": self.state.filename,
222
+ "rows": len(self.state.df),
223
+ "cols": len(self.state.df.columns),
224
+ })
225
+
226
+ for i, step in enumerate(self.state.steps):
227
+ if self.state.cancelled:
228
+ yield self._event("cancelled", {"step": step.id})
229
+ return
230
+
231
+ self.state.current_step_index = i
232
+ step.status = StepStatus.RUNNING
233
+ step.started_at = time.time()
234
+
235
+ yield self._event("step_start", {
236
+ "step_index": i,
237
+ "step": step.to_dict()
238
+ })
239
+
240
+ try:
241
+ result = await self._execute_step(step)
242
+ step.status = StepStatus.COMPLETED
243
+ step.completed_at = time.time()
244
+ step.result = result
245
+ step.progress = 100.0
246
+
247
+ yield self._event("step_complete", {
248
+ "step_index": i,
249
+ "step": step.to_dict()
250
+ })
251
+
252
+ except Exception as e:
253
+ step.status = StepStatus.FAILED
254
+ step.completed_at = time.time()
255
+ step.error = str(e)
256
+ logger.error(f"Autopilot step {step.id} failed: {e}")
257
+ traceback.print_exc()
258
+
259
+ yield self._event("step_error", {
260
+ "step_index": i,
261
+ "step": step.to_dict(),
262
+ "error": str(e)
263
+ })
264
+ # Continue with next step despite error
265
+ continue
266
+
267
+ self.state.completed_at = time.time()
268
+ elapsed = round(self.state.completed_at - self.state.started_at, 1)
269
+
270
+ session_payload = {
271
+ "session_id": self.state.session_id,
272
+ "elapsed_seconds": elapsed,
273
+ "insights_count": len(self.state.insights),
274
+ "charts_count": len(self.state.charts),
275
+ "model_trained": self.state.model_result is not None,
276
+ "model_result": self.state.model_result,
277
+ "data_profile": self.state.steps[1].result if len(self.state.steps) > 1 and self.state.steps[1].result else None,
278
+ "insights": self.state.insights,
279
+ "summary": self.state.report or "Analysis complete.",
280
+ }
281
+
282
+ yield self._event("session_complete", session_payload)
283
+
284
+ # Fire webhooks asynchronously without blocking the stream
285
+ asyncio.create_task(self._trigger_webhooks(session_payload))
286
+
287
+ async def _trigger_webhooks(self, payload: dict):
288
+ """Fire HTTP requests to the user's registered webhooks."""
289
+ try:
290
+ from database.db import AsyncSessionLocal
291
+ from database.orm import Webhook
292
+ from sqlalchemy import select
293
+ import httpx
294
+
295
+ async with AsyncSessionLocal() as db:
296
+ result = await db.execute(
297
+ select(Webhook).filter(
298
+ Webhook.user_id == self.state.user_id,
299
+ Webhook.status == "active"
300
+ )
301
+ )
302
+ webhooks = result.scalars().all()
303
+
304
+ user_webhooks = [
305
+ w for w in webhooks
306
+ if "autopilot.completed" in (w.events or [])
307
+ ]
308
+
309
+ if not user_webhooks:
310
+ return
311
+
312
+ async with httpx.AsyncClient() as client:
313
+ for wh in user_webhooks:
314
+ try:
315
+ await client.post(
316
+ wh.url,
317
+ json={"event": "autopilot.completed", "data": payload},
318
+ timeout=10.0
319
+ )
320
+ logger.info(f"Successfully fired webhook to {wh.url}")
321
+ except Exception as e:
322
+ logger.warning(f"Webhook {wh.url} failed: {e}")
323
+ except Exception as e:
324
+ logger.error(f"Failed to process webhooks: {e}")
325
+
326
+ async def _execute_step(self, step: AutopilotStep) -> Dict[str, Any]:
327
+ """Route execution to the correct handler."""
328
+ handlers = {
329
+ AutopilotPhase.PLANNING: self._step_planning,
330
+ AutopilotPhase.DATA_PROFILING: self._step_profiling,
331
+ AutopilotPhase.DATA_CLEANING: self._step_cleaning,
332
+ AutopilotPhase.FEATURE_DISCOVERY: self._step_features,
333
+ AutopilotPhase.INSIGHT_EXTRACTION: self._step_insights,
334
+ AutopilotPhase.VISUALIZATION: self._step_visualization,
335
+ AutopilotPhase.MODEL_TRAINING: self._step_training,
336
+ AutopilotPhase.MODEL_EVALUATION: self._step_evaluation,
337
+ AutopilotPhase.CODE_GENERATION: self._step_code_generation,
338
+ AutopilotPhase.REPORT_GENERATION: self._step_report,
339
+ }
340
+ handler = handlers.get(step.phase)
341
+ if handler:
342
+ # Add a small artificial delay so the user can see the AI "thinking"
343
+ # and the UI animations have time to run smoothly.
344
+ await asyncio.sleep(1.5)
345
+ # Run the handler
346
+ return await handler(step)
347
+ return {"status": "skipped"}
348
+
349
+ # =========================================================================
350
+ # STEP HANDLERS
351
+ # =========================================================================
352
+
353
+ async def _step_planning(self, step: AutopilotStep) -> Dict:
354
+ """Analyze goal and data to create intelligent plan."""
355
+ df = self.state.df
356
+
357
+ # Auto-detect target column if not provided
358
+ if not self.state.target_column:
359
+ # Heuristics: last column, or columns with names like "target", "label", "class", "price", "sales"
360
+ target_keywords = ['target', 'label', 'class', 'churn', 'fraud', 'price', 'sales',
361
+ 'revenue', 'profit', 'rating', 'score', 'outcome', 'result',
362
+ 'survival', 'default', 'response', 'converted', 'purchased']
363
+
364
+ for col in df.columns:
365
+ if any(kw in col.lower() for kw in target_keywords):
366
+ self.state.target_column = col
367
+ break
368
+
369
+ if not self.state.target_column:
370
+ # Use last numeric column
371
+ numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
372
+ if numeric_cols:
373
+ self.state.target_column = numeric_cols[-1]
374
+ else:
375
+ self.state.target_column = df.columns[-1]
376
+
377
+ # Detect task type
378
+ target_col = self.state.target_column
379
+ if target_col in df.columns:
380
+ nunique = df[target_col].nunique()
381
+ dtype = df[target_col].dtype
382
+
383
+ if dtype == 'object' or (dtype in ['int64', 'float64'] and nunique <= 20 and nunique / len(df) < 0.05):
384
+ self.state.task_type = "classification"
385
+ else:
386
+ self.state.task_type = "regression"
387
+ else:
388
+ self.state.task_type = "regression"
389
+
390
+ plan_summary = (
391
+ f"📋 **Analysis Plan:**\n"
392
+ f"- Dataset: {self.state.filename} ({len(df):,} rows × {len(df.columns)} cols)\n"
393
+ f"- Target: `{self.state.target_column}`\n"
394
+ f"- Task: {self.state.task_type}\n"
395
+ f"- Goal: {self.state.goal}\n"
396
+ f"- Strategy: Full autonomous pipeline — profile → clean → engineer → train → deploy"
397
+ )
398
+
399
+ self.state.insights.append(plan_summary)
400
+
401
+ return {
402
+ "target_column": self.state.target_column,
403
+ "task_type": self.state.task_type,
404
+ "rows": len(df),
405
+ "cols": len(df.columns),
406
+ "plan": plan_summary
407
+ }
408
+
409
+ async def _step_profiling(self, step: AutopilotStep) -> Dict:
410
+ """Deep data profiling."""
411
+ df = self.state.df
412
+
413
+ profile = {
414
+ "shape": {"rows": len(df), "cols": len(df.columns)},
415
+ "dtypes": {col: str(dtype) for col, dtype in df.dtypes.items()},
416
+ "missing": {col: int(df[col].isnull().sum()) for col in df.columns if df[col].isnull().sum() > 0},
417
+ "missing_pct": round(df.isnull().sum().sum() / (len(df) * len(df.columns)) * 100, 2),
418
+ "duplicates": int(df.duplicated().sum()),
419
+ "numeric_cols": df.select_dtypes(include=[np.number]).columns.tolist(),
420
+ "categorical_cols": df.select_dtypes(include=['object', 'category']).columns.tolist(),
421
+ "memory_mb": round(df.memory_usage(deep=True).sum() / 1024 / 1024, 2),
422
+ }
423
+
424
+ # Compute basic stats for numerics
425
+ numeric_stats = {}
426
+ for col in profile["numeric_cols"][:15]: # Limit to 15 cols for speed
427
+ numeric_stats[col] = {
428
+ "mean": round(float(df[col].mean()), 3) if not df[col].isnull().all() else None,
429
+ "std": round(float(df[col].std()), 3) if not df[col].isnull().all() else None,
430
+ "min": float(df[col].min()) if not df[col].isnull().all() else None,
431
+ "max": float(df[col].max()) if not df[col].isnull().all() else None,
432
+ }
433
+ profile["numeric_stats"] = numeric_stats
434
+
435
+ # Generate profiling insight
436
+ insight = (
437
+ f"🔍 **Data Profile:** {len(df):,} rows × {len(df.columns)} cols | "
438
+ f"{len(profile['numeric_cols'])} numeric, {len(profile['categorical_cols'])} categorical | "
439
+ f"{profile['missing_pct']}% missing | {profile['duplicates']} duplicates | "
440
+ f"{profile['memory_mb']} MB"
441
+ )
442
+ self.state.insights.append(insight)
443
+
444
+ return profile
445
+
446
+ async def _step_cleaning(self, step: AutopilotStep) -> Dict:
447
+ """Autonomous data cleaning."""
448
+ df = self.state.df.copy()
449
+ fixes = []
450
+
451
+ original_shape = df.shape
452
+
453
+ # 1. Remove duplicates
454
+ dup_count = df.duplicated().sum()
455
+ if dup_count > 0:
456
+ df = df.drop_duplicates()
457
+ fixes.append(f"Removed {dup_count} duplicate rows")
458
+
459
+ # 2. Handle missing values
460
+ for col in df.columns:
461
+ missing = df[col].isnull().sum()
462
+ if missing > 0:
463
+ pct = missing / len(df) * 100
464
+ if pct > 60:
465
+ df = df.drop(columns=[col])
466
+ fixes.append(f"Dropped column `{col}` ({pct:.0f}% missing)")
467
+ elif df[col].dtype in ['float64', 'int64']:
468
+ median_val = df[col].median()
469
+ df[col] = df[col].fillna(median_val)
470
+ fixes.append(f"Filled {missing} missing values in `{col}` with median ({median_val:.2f})")
471
+ else:
472
+ mode_val = df[col].mode()[0] if len(df[col].mode()) > 0 else "Unknown"
473
+ df[col] = df[col].fillna(mode_val)
474
+ fixes.append(f"Filled {missing} missing values in `{col}` with mode ({mode_val})")
475
+
476
+ self.state.df = df
477
+
478
+ insight = f"🧹 **Data Cleaning:** Applied {len(fixes)} fixes. Shape: {original_shape} → {df.shape}"
479
+ self.state.insights.append(insight)
480
+
481
+ return {"fixes": fixes, "original_shape": list(original_shape), "new_shape": list(df.shape)}
482
+
483
+ async def _step_features(self, step: AutopilotStep) -> Dict:
484
+ """Feature discovery and engineering."""
485
+ df = self.state.df
486
+ new_features = []
487
+
488
+ # Auto-detect and engineer date features
489
+ for col in df.columns:
490
+ if df[col].dtype == 'object':
491
+ try:
492
+ parsed = pd.to_datetime(df[col], errors='coerce')
493
+ if parsed.notna().sum() > len(df) * 0.5:
494
+ df[f'{col}_year'] = parsed.dt.year
495
+ df[f'{col}_month'] = parsed.dt.month
496
+ df[f'{col}_dayofweek'] = parsed.dt.dayofweek
497
+ new_features.extend([f'{col}_year', f'{col}_month', f'{col}_dayofweek'])
498
+ df = df.drop(columns=[col])
499
+ except:
500
+ pass
501
+
502
+ self.state.df = df
503
+
504
+ # Compute correlations with target
505
+ target = self.state.target_column
506
+ correlations = {}
507
+ if target in df.columns and df[target].dtype in ['float64', 'int64']:
508
+ numeric_df = df.select_dtypes(include=[np.number])
509
+ if target in numeric_df.columns:
510
+ corrs = numeric_df.corr()[target].drop(target, errors='ignore').abs().sort_values(ascending=False)
511
+ correlations = {col: round(float(val), 3) for col, val in corrs.head(10).items()}
512
+
513
+ insight = f"⚡ **Features:** Engineered {len(new_features)} new features. Top correlations with `{target}`: {', '.join(f'{k}({v})' for k, v in list(correlations.items())[:5])}"
514
+ self.state.insights.append(insight)
515
+
516
+ return {
517
+ "new_features": new_features,
518
+ "top_correlations": correlations,
519
+ "total_features": len(df.columns)
520
+ }
521
+
522
+ async def _step_insights(self, step: AutopilotStep) -> Dict:
523
+ """Extract key data insights using statistical analysis."""
524
+ df = self.state.df
525
+ key_insights = []
526
+
527
+ # 1. Distribution insights
528
+ numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
529
+ for col in numeric_cols[:10]:
530
+ skewness = df[col].skew()
531
+ if abs(skewness) > 1:
532
+ direction = "right" if skewness > 0 else "left"
533
+ key_insights.append(f"📊 `{col}` is heavily {direction}-skewed (skew={skewness:.2f})")
534
+
535
+ # 2. Categorical insights
536
+ cat_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
537
+ for col in cat_cols[:5]:
538
+ top_val = df[col].value_counts().head(1)
539
+ if len(top_val) > 0:
540
+ pct = top_val.values[0] / len(df) * 100
541
+ if pct > 50:
542
+ key_insights.append(f"⚠️ `{col}` is dominated by '{top_val.index[0]}' ({pct:.0f}% of data)")
543
+
544
+ # 3. Correlation insights
545
+ if len(numeric_cols) > 1:
546
+ corr_matrix = df[numeric_cols].corr()
547
+ for i in range(len(numeric_cols)):
548
+ for j in range(i+1, len(numeric_cols)):
549
+ corr_val = corr_matrix.iloc[i, j]
550
+ if abs(corr_val) > 0.8:
551
+ key_insights.append(
552
+ f"🔗 Strong {'positive' if corr_val > 0 else 'negative'} correlation between "
553
+ f"`{numeric_cols[i]}` and `{numeric_cols[j]}` ({corr_val:.2f})"
554
+ )
555
+
556
+ self.state.insights.extend(key_insights[:10]) # Cap at 10
557
+
558
+ return {"insights": key_insights[:10], "count": len(key_insights)}
559
+
560
+ async def _step_visualization(self, step: AutopilotStep) -> Dict:
561
+ """Generate key visualizations."""
562
+ df = self.state.df
563
+ charts = []
564
+
565
+ target = self.state.target_column
566
+ numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
567
+
568
+ # 1. Target distribution chart
569
+ if target in df.columns:
570
+ if self.state.task_type == "classification":
571
+ value_counts = df[target].value_counts().head(10)
572
+ charts.append({
573
+ "type": "bar",
574
+ "title": f"Distribution of {target}",
575
+ "data": {"labels": value_counts.index.tolist(), "values": value_counts.values.tolist()},
576
+ "description": f"Class distribution of target variable `{target}`"
577
+ })
578
+ else:
579
+ charts.append({
580
+ "type": "histogram",
581
+ "title": f"Distribution of {target}",
582
+ "data": {"values": df[target].dropna().tolist()[:1000]},
583
+ "description": f"Histogram of target variable `{target}`"
584
+ })
585
+
586
+ # 2. Correlation heatmap data
587
+ if len(numeric_cols) > 1:
588
+ top_cols = numeric_cols[:10]
589
+ corr = df[top_cols].corr().round(3)
590
+ charts.append({
591
+ "type": "heatmap",
592
+ "title": "Feature Correlation Matrix",
593
+ "data": {
594
+ "labels": top_cols,
595
+ "values": corr.values.tolist()
596
+ },
597
+ "description": "Correlation heatmap of top numeric features"
598
+ })
599
+
600
+ # 3. Top feature bar chart
601
+ if target in df.columns and target in numeric_cols and len(numeric_cols) > 1:
602
+ corrs = df[numeric_cols].corr()[target].drop(target, errors='ignore').abs().sort_values(ascending=False).head(8)
603
+ charts.append({
604
+ "type": "bar",
605
+ "title": f"Top Features Correlated with {target}",
606
+ "data": {"labels": corrs.index.tolist(), "values": corrs.values.tolist()},
607
+ "description": "Features most correlated with the target variable"
608
+ })
609
+
610
+ self.state.charts = charts
611
+
612
+ return {"charts_generated": len(charts)}
613
+
614
+ async def _step_training(self, step: AutopilotStep) -> Dict:
615
+ """Train ML model using the existing God-Level AutoML engine."""
616
+ try:
617
+ from ml.god_level_automl import run_god_level
618
+ import asyncio
619
+
620
+ # Run the heavy blocking AutoML training in a separate thread
621
+ # to prevent starving the event loop and freezing the SSE stream.
622
+ result = await asyncio.to_thread(
623
+ run_god_level,
624
+ df=self.state.df,
625
+ target_column=self.state.target_column,
626
+ user_id=self.state.user_id,
627
+ mode="balanced"
628
+ )
629
+
630
+ self.state.model_result = {
631
+ "best_model": result.best_model_name,
632
+ "metrics": result.best_model_metrics,
633
+ "reliability": result.best_model_reliability,
634
+ "leaderboard": result.leaderboard[:5] if result.leaderboard else [],
635
+ "processing_time": result.processing_time,
636
+ }
637
+
638
+ insight = (
639
+ f"🏆 **Best Model: {result.best_model_name}** | "
640
+ f"Reliability: {result.best_model_reliability:.1f}/100 | "
641
+ f"Trained in {result.processing_time:.1f}s"
642
+ )
643
+ self.state.insights.append(insight)
644
+
645
+ return self.state.model_result
646
+
647
+ except Exception as e:
648
+ logger.error(f"Model training failed: {e}")
649
+ # Fallback to simple sklearn
650
+ return await self._fallback_training(step)
651
+
652
+ async def _fallback_training(self, step: AutopilotStep) -> Dict:
653
+ """Fallback training with basic sklearn if God-Level fails."""
654
+ from sklearn.model_selection import train_test_split, cross_val_score
655
+ from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
656
+ from sklearn.preprocessing import LabelEncoder
657
+ from sklearn.metrics import accuracy_score, r2_score, mean_squared_error
658
+
659
+ df = self.state.df.copy()
660
+ target = self.state.target_column
661
+
662
+ if target not in df.columns:
663
+ return {"error": f"Target column '{target}' not found"}
664
+
665
+ # Prepare data
666
+ X = df.drop(columns=[target])
667
+ y = df[target]
668
+
669
+ # Encode categoricals
670
+ encoders = {}
671
+ for col in X.select_dtypes(include=['object']).columns:
672
+ le = LabelEncoder()
673
+ X[col] = le.fit_transform(X[col].astype(str))
674
+ encoders[col] = le
675
+
676
+ if y.dtype == 'object':
677
+ le_target = LabelEncoder()
678
+ y = le_target.fit_transform(y)
679
+
680
+ # Drop remaining non-numeric
681
+ X = X.select_dtypes(include=[np.number])
682
+
683
+ # Fill any remaining NaN
684
+ X = X.fillna(0)
685
+
686
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
687
+
688
+ if self.state.task_type == "classification":
689
+ model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
690
+ model.fit(X_train, y_train)
691
+ y_pred = model.predict(X_test)
692
+ score = accuracy_score(y_test, y_pred)
693
+ metric_name = "accuracy"
694
+ else:
695
+ model = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1)
696
+ model.fit(X_train, y_train)
697
+ y_pred = model.predict(X_test)
698
+ score = r2_score(y_test, y_pred)
699
+ metric_name = "r2"
700
+
701
+ # Save model
702
+ try:
703
+ from ml.model_persistence import model_persistence
704
+ model_persistence.save_model(
705
+ user_id=self.state.user_id,
706
+ model=model,
707
+ metadata={
708
+ "model_name": "RandomForest",
709
+ "task_type": self.state.task_type,
710
+ "target_column": self.state.target_column,
711
+ "feature_columns": X.columns.tolist(),
712
+ metric_name: round(score, 4),
713
+ "source": "autopilot"
714
+ }
715
+ )
716
+ except Exception as save_err:
717
+ logger.warning(f"Could not save model: {save_err}")
718
+
719
+ self.state.model_result = {
720
+ "best_model": "RandomForest",
721
+ "metrics": {metric_name: round(score, 4)},
722
+ "reliability": round(score * 100, 1),
723
+ "processing_time": round(time.time() - step.started_at, 1),
724
+ }
725
+
726
+ insight = f"🏆 **Best Model: RandomForest** | {metric_name}: {score:.4f}"
727
+ self.state.insights.append(insight)
728
+
729
+ return self.state.model_result
730
+
731
+ async def _step_evaluation(self, step: AutopilotStep) -> Dict:
732
+ """Evaluate model and generate explanations."""
733
+ if not self.state.model_result:
734
+ return {"status": "skipped", "reason": "No model trained"}
735
+
736
+ eval_result = {
737
+ "model": self.state.model_result.get("best_model"),
738
+ "metrics": self.state.model_result.get("metrics", {}),
739
+ "reliability": self.state.model_result.get("reliability", 0),
740
+ "verdict": "✅ Model is production-ready" if self.state.model_result.get("reliability", 0) > 70 else "⚠️ Model needs improvement"
741
+ }
742
+
743
+ self.state.insights.append(eval_result["verdict"])
744
+ return eval_result
745
+
746
+ async def _step_code_generation(self, step: AutopilotStep) -> Dict:
747
+ """Generate deployment code."""
748
+ if not self.state.model_result:
749
+ return {"status": "skipped", "reason": "No model to deploy"}
750
+
751
+ try:
752
+ from ml.ml_code_generator import generate_code_zip
753
+ from config.settings import Settings
754
+ from ml.model_persistence import model_persistence
755
+
756
+ base_storage = Settings.STORAGE
757
+ model_path = base_storage / "models" / self.state.user_id / "active_model.pkl"
758
+
759
+ if model_path.exists():
760
+ meta_path = model_path.parent / "active_metadata.json"
761
+ metadata = {}
762
+ if meta_path.exists():
763
+ with open(meta_path, 'r') as f:
764
+ metadata = json.load(f)
765
+
766
+ buf = generate_code_zip(model_path, metadata)
767
+
768
+ self.state.insights.append("💻 **Deployment code generated** — Ready to run as a local API server")
769
+ return {"status": "generated", "files_count": 5}
770
+ else:
771
+ return {"status": "skipped", "reason": "Model file not found at expected path"}
772
+ except Exception as e:
773
+ logger.warning(f"Code generation failed: {e}")
774
+ return {"status": "error", "error": str(e)}
775
+
776
+ async def _step_report(self, step: AutopilotStep) -> Dict:
777
+ """Generate the final analysis report."""
778
+ elapsed = round(time.time() - self.state.started_at, 1)
779
+
780
+ report_lines = [
781
+ f"# 🚀 DataVision Autopilot Report",
782
+ f"",
783
+ f"**Dataset:** {self.state.filename}",
784
+ f"**Goal:** {self.state.goal}",
785
+ f"**Target:** `{self.state.target_column}` ({self.state.task_type})",
786
+ f"**Total Time:** {elapsed}s",
787
+ f"",
788
+ f"---",
789
+ f"",
790
+ f"## Key Insights",
791
+ f"",
792
+ ]
793
+
794
+ for i, insight in enumerate(self.state.insights):
795
+ report_lines.append(f"{i+1}. {insight}")
796
+
797
+ if self.state.model_result:
798
+ report_lines.extend([
799
+ f"",
800
+ f"## Model Performance",
801
+ f"",
802
+ f"- **Best Model:** {self.state.model_result.get('best_model', 'N/A')}",
803
+ f"- **Metrics:** {json.dumps(self.state.model_result.get('metrics', {}), indent=2)}",
804
+ f"- **Reliability:** {self.state.model_result.get('reliability', 0)}/100",
805
+ ])
806
+
807
+ report_lines.extend([
808
+ f"",
809
+ f"---",
810
+ f"*Generated by DataVision Agentic Autopilot v5.0*",
811
+ ])
812
+
813
+ self.state.report = "\n".join(report_lines)
814
+ return {"report": self.state.report}
815
+
816
+ def _event(self, event_type: str, data: Dict) -> Dict:
817
+ """Create a standardized event dict."""
818
+ return {
819
+ "type": event_type,
820
+ "timestamp": datetime.utcnow().isoformat(),
821
+ "data": data
822
+ }
823
+
824
+ def cancel(self):
825
+ """Cancel the running autopilot."""
826
+ self.state.cancelled = True
827
+
828
+
829
+ # Global registry of active autopilot sessions
830
+ active_sessions: Dict[str, AgenticAutopilot] = {}
backend/agents/base.py ADDED
@@ -0,0 +1,284 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ 🤖 Agentic AutoML - Base Agent Framework
3
+
4
+ This module defines the foundational components for the multi-agent system:
5
+ - Message types for agent communication
6
+ - Base Agent class with lifecycle methods
7
+ - Agent result and status types
8
+ """
9
+
10
+ from abc import ABC, abstractmethod
11
+ from dataclasses import dataclass, field
12
+ from enum import Enum
13
+ from typing import Any, Dict, List, Optional, Callable
14
+ from datetime import datetime
15
+ from agents.memory import AgentMemory
16
+ import uuid
17
+ import logging
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+
22
+ # =============================================================================
23
+ # MESSAGE TYPES - For inter-agent communication
24
+ # =============================================================================
25
+
26
+ class MessageType(Enum):
27
+ """Types of messages agents can send"""
28
+ # Decision messages
29
+ APPROVAL = "approval"
30
+ REJECTION = "rejection"
31
+ RETRY = "retry"
32
+
33
+ # Signal messages
34
+ ESCALATE = "escalate"
35
+ REQUEST_FEEDBACK = "request_feedback"
36
+ PHASE_COMPLETE = "phase_complete"
37
+
38
+ # Control messages
39
+ ABORT = "abort"
40
+ PAUSE = "pause"
41
+ RESUME = "resume"
42
+
43
+
44
+ class AgentStatus(Enum):
45
+ """Agent execution status"""
46
+ IDLE = "idle"
47
+ RUNNING = "running"
48
+ SUCCESS = "success"
49
+ FAILED = "failed"
50
+ WAITING = "waiting"
51
+ RETRY = "retry"
52
+
53
+
54
+ class Phase(Enum):
55
+ """Pipeline phases"""
56
+ FAST_DISCOVERY = "fast_discovery"
57
+ DEEP_VALIDATION = "deep_validation"
58
+
59
+
60
+ @dataclass
61
+ class AgentMessage:
62
+ """Message passed between agents"""
63
+ id: str = field(default_factory=lambda: str(uuid.uuid4())[:8])
64
+ sender: str = ""
65
+ receiver: str = ""
66
+ type: MessageType = MessageType.APPROVAL
67
+ payload: Dict[str, Any] = field(default_factory=dict)
68
+ timestamp: datetime = field(default_factory=datetime.now)
69
+
70
+ def to_dict(self) -> Dict:
71
+ return {
72
+ "id": self.id,
73
+ "sender": self.sender,
74
+ "receiver": self.receiver,
75
+ "type": self.type.value,
76
+ "payload": self.payload,
77
+ "timestamp": self.timestamp.isoformat()
78
+ }
79
+
80
+
81
+ @dataclass
82
+ class AgentResult:
83
+ """Result returned by an agent after execution"""
84
+ status: AgentStatus
85
+ agent_name: str
86
+ phase: Phase
87
+ data: Dict[str, Any] = field(default_factory=dict)
88
+ messages: List[AgentMessage] = field(default_factory=list)
89
+ errors: List[str] = field(default_factory=list)
90
+ recommendations: List[str] = field(default_factory=list)
91
+ metrics: Dict[str, float] = field(default_factory=dict)
92
+ duration_seconds: float = 0.0
93
+
94
+ @property
95
+ def success(self) -> bool:
96
+ return self.status == AgentStatus.SUCCESS
97
+
98
+ @property
99
+ def should_retry(self) -> bool:
100
+ return self.status == AgentStatus.RETRY
101
+
102
+ def add_message(self, receiver: str, msg_type: MessageType, payload: Dict = None):
103
+ """Helper to add a message to send"""
104
+ self.messages.append(AgentMessage(
105
+ sender=self.agent_name,
106
+ receiver=receiver,
107
+ type=msg_type,
108
+ payload=payload or {}
109
+ ))
110
+
111
+
112
+ # =============================================================================
113
+ # BASE AGENT CLASS
114
+ # =============================================================================
115
+
116
+ class BaseAgent(ABC):
117
+ """
118
+ Base class for all agents in the Agentic AutoML system.
119
+
120
+ Each agent has:
121
+ - A name and description
122
+ - Access to shared memory
123
+ - Ability to send/receive messages
124
+ - Lifecycle methods (validate, execute, handle_failure)
125
+ """
126
+
127
+ name: str = "BaseAgent"
128
+ description: str = "Base agent class"
129
+
130
+ def __init__(self, memory: 'AgentMemory' = None):
131
+ self.memory = memory
132
+ self.current_phase = Phase.FAST_DISCOVERY
133
+ self.retry_count = 0
134
+ self.max_retries = 3
135
+ self._start_time = None
136
+ self.logger = logging.getLogger(f"agent.{self.name}")
137
+
138
+ # =========================================================================
139
+ # LIFECYCLE METHODS
140
+ # =========================================================================
141
+
142
+ def run(self, **kwargs) -> AgentResult:
143
+ """
144
+ Main entry point for agent execution.
145
+ Handles timing, error catching, and retry logic.
146
+ """
147
+ import time
148
+ self._start_time = time.time()
149
+
150
+ try:
151
+ # Pre-execution validation
152
+ validation = self.validate(**kwargs)
153
+ if not validation['valid']:
154
+ return AgentResult(
155
+ status=AgentStatus.FAILED,
156
+ agent_name=self.name,
157
+ phase=self.current_phase,
158
+ errors=validation.get('errors', ['Validation failed']),
159
+ duration_seconds=time.time() - self._start_time
160
+ )
161
+
162
+ # Main execution
163
+ self.logger.info(f"🤖 {self.name} starting...")
164
+ result = self.execute(**kwargs)
165
+ result.duration_seconds = time.time() - self._start_time
166
+
167
+ # Log result
168
+ if result.success:
169
+ self.logger.info(f"✅ {self.name} completed in {result.duration_seconds:.2f}s")
170
+ else:
171
+ self.logger.warning(f"⚠️ {self.name} finished with status: {result.status.value}")
172
+
173
+ return result
174
+
175
+ except Exception as e:
176
+ self.logger.error(f"❌ {self.name} error: {str(e)}")
177
+ return self.handle_failure(e, **kwargs)
178
+
179
+ def validate(self, **kwargs) -> Dict[str, Any]:
180
+ """
181
+ Validate inputs before execution.
182
+ Override in subclasses for specific validation.
183
+ """
184
+ return {'valid': True}
185
+
186
+ @abstractmethod
187
+ def execute(self, **kwargs) -> AgentResult:
188
+ """
189
+ Main execution logic. Must be implemented by subclasses.
190
+ """
191
+ pass
192
+
193
+ def handle_failure(self, error: Exception, **kwargs) -> AgentResult:
194
+ """
195
+ Handle execution failures. Can be overridden for custom recovery.
196
+ """
197
+ import time
198
+ self.retry_count += 1
199
+
200
+ if self.retry_count < self.max_retries:
201
+ return AgentResult(
202
+ status=AgentStatus.RETRY,
203
+ agent_name=self.name,
204
+ phase=self.current_phase,
205
+ errors=[str(error)],
206
+ recommendations=[f"Retry attempt {self.retry_count}/{self.max_retries}"],
207
+ duration_seconds=time.time() - self._start_time if self._start_time else 0
208
+ )
209
+
210
+ return AgentResult(
211
+ status=AgentStatus.FAILED,
212
+ agent_name=self.name,
213
+ phase=self.current_phase,
214
+ errors=[str(error), f"Max retries ({self.max_retries}) exceeded"],
215
+ duration_seconds=time.time() - self._start_time if self._start_time else 0
216
+ )
217
+
218
+ # =========================================================================
219
+ # PHASE CONTROL
220
+ # =========================================================================
221
+
222
+ def set_phase(self, phase: Phase):
223
+ """Set the current execution phase"""
224
+ self.current_phase = phase
225
+ self.logger.info(f"📍 Phase set to: {phase.value}")
226
+
227
+ def is_fast_phase(self) -> bool:
228
+ """Check if in fast discovery phase"""
229
+ return self.current_phase == Phase.FAST_DISCOVERY
230
+
231
+ def is_deep_phase(self) -> bool:
232
+ """Check if in deep validation phase"""
233
+ return self.current_phase == Phase.DEEP_VALIDATION
234
+
235
+ # =========================================================================
236
+ # MEMORY ACCESS
237
+ # =========================================================================
238
+
239
+ def read_state(self, key: str, default: Any = None) -> Any:
240
+ """Read from shared memory"""
241
+ if self.memory:
242
+ return self.memory.get(key, default)
243
+ return default
244
+
245
+ def write_state(self, key: str, value: Any, stage: str = None):
246
+ """Write to shared memory (immutable per stage)"""
247
+ if self.memory:
248
+ self.memory.set(key, value, stage or self.name)
249
+
250
+ def get_artifacts(self, artifact_type: str) -> List[Any]:
251
+ """Get artifacts of a specific type"""
252
+ if self.memory:
253
+ return self.memory.get_artifacts(artifact_type)
254
+ return []
255
+
256
+
257
+ # =============================================================================
258
+ # AGENT REGISTRY
259
+ # =============================================================================
260
+
261
+ class AgentRegistry:
262
+ """Registry for managing agent instances"""
263
+
264
+ _agents: Dict[str, BaseAgent] = {}
265
+
266
+ @classmethod
267
+ def register(cls, agent: BaseAgent):
268
+ """Register an agent"""
269
+ cls._agents[agent.name] = agent
270
+
271
+ @classmethod
272
+ def get(cls, name: str) -> Optional[BaseAgent]:
273
+ """Get an agent by name"""
274
+ return cls._agents.get(name)
275
+
276
+ @classmethod
277
+ def all(cls) -> List[BaseAgent]:
278
+ """Get all registered agents"""
279
+ return list(cls._agents.values())
280
+
281
+ @classmethod
282
+ def clear(cls):
283
+ """Clear all registered agents"""
284
+ cls._agents.clear()
backend/agents/base/agent_runner.py ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Base Agent Runner - Abstract class for all AI agents
3
+ Provides common functionality for insight detection and notification dispatch.
4
+ """
5
+
6
+ from abc import ABC, abstractmethod
7
+ from typing import List, Dict, Any, Optional
8
+ from datetime import datetime
9
+ import uuid
10
+ import logging
11
+ import json
12
+
13
+ from sqlalchemy.ext.asyncio import AsyncSession
14
+ from sqlalchemy.future import select
15
+
16
+ from database.db import AsyncSessionLocal
17
+ from database.orm import UserProfile, WorkspaceMember
18
+ from database.workspace_ops import get_workspace_members
19
+ from services.notification_service import (
20
+ enqueue_notification,
21
+ NotificationJob,
22
+ should_notify,
23
+ in_dnd_window,
24
+ check_rate_limit
25
+ )
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+
30
+ class Insight:
31
+ """Data class for AI-generated insights"""
32
+ def __init__(
33
+ self,
34
+ title: str,
35
+ body: str,
36
+ severity: str, # 'low' | 'medium' | 'high'
37
+ score: float = None,
38
+ metadata: Dict[str, Any] = None,
39
+ chart_payload: Dict[str, Any] = None
40
+ ):
41
+ self.title = title
42
+ self.body = body
43
+ self.severity = severity
44
+ self.score = score
45
+ self.metadata = metadata or {}
46
+ self.chart_payload = chart_payload
47
+
48
+
49
+ class AgentRunner(ABC):
50
+ """Base class for all AI agents"""
51
+
52
+ def __init__(self, agent_name: str):
53
+ self.agent_name = agent_name
54
+ self.logger = logging.getLogger(f"agents.{agent_name}")
55
+
56
+ @abstractmethod
57
+ async def detect_insights(self, workspace_id: str) -> List[Insight]:
58
+ """
59
+ Detect insights for a workspace
60
+ Must be implemented by each agent
61
+ """
62
+ pass
63
+
64
+ async def run(self, workspace_id: str):
65
+ """
66
+ Main execution method
67
+ 1. Detect insights
68
+ 2. Persist to DB
69
+ 3. Log execution
70
+ 4. Dispatch notifications
71
+ """
72
+ self.logger.info(f"Running {self.agent_name} for workspace {workspace_id}")
73
+
74
+ try:
75
+ # Log agent start
76
+ await self._log_execution(workspace_id, 'started', f"{self.agent_name} execution started")
77
+
78
+ # Detect insights
79
+ insights = await self.detect_insights(workspace_id)
80
+
81
+ self.logger.info(f"Detected {len(insights)} insights for workspace {workspace_id}")
82
+
83
+ for insight in insights:
84
+ await self._process_insight(workspace_id, insight)
85
+
86
+ # Log agent completion
87
+ await self._log_execution(
88
+ workspace_id,
89
+ 'completed',
90
+ f"{self.agent_name} completed: {len(insights)} insights generated"
91
+ )
92
+
93
+ except Exception as e:
94
+ self.logger.error(f"Agent execution failed: {e}", exc_info=True)
95
+
96
+ # Log error
97
+ await self._log_execution(
98
+ workspace_id,
99
+ 'error',
100
+ f"{self.agent_name} failed: {str(e)}",
101
+ {'error': str(e)}
102
+ )
103
+
104
+ async def _process_insight(self, workspace_id: str, insight: Insight):
105
+ """Process a single insight: persist, log, and notify"""
106
+ try:
107
+ # 1. Persist insight to database
108
+ saved_insight = await self._persist_insight(workspace_id, insight)
109
+
110
+ if not saved_insight:
111
+ self.logger.error("Failed to persist insight")
112
+ return
113
+
114
+ insight_id = saved_insight['id']
115
+
116
+ # 2. Log insight creation
117
+ await self._log_execution(
118
+ workspace_id,
119
+ 'insight_created',
120
+ f"Insight created: {insight.title}",
121
+ {'insight_id': insight_id, 'severity': insight.severity}
122
+ )
123
+
124
+ # 3. Dispatch notifications to workspace members
125
+ await self._dispatch_notifications(workspace_id, insight_id, insight)
126
+
127
+ except Exception as e:
128
+ self.logger.error(f"Failed to process insight: {e}", exc_info=True)
129
+
130
+ async def _persist_insight(self, workspace_id: str, insight: Insight) -> Optional[Dict[str, Any]]:
131
+ """Save insight to database via direct SQL (ai_insights table)"""
132
+ try:
133
+ async with AsyncSessionLocal() as db:
134
+ # Use raw SQL insert since we don't have a dedicated AIInsight ORM model yet
135
+ from sqlalchemy import text
136
+ insight_id = str(uuid.uuid4())
137
+ stmt = text("""
138
+ INSERT INTO ai_insights (id, workspace_id, title, body, severity, score, metadata, chart_payload, created_by_agent, created_at)
139
+ VALUES (:id, :workspace_id, :title, :body, :severity, :score, :metadata, :chart_payload, :agent, :created_at)
140
+ ON CONFLICT DO NOTHING
141
+ RETURNING id
142
+ """)
143
+ result = await db.execute(stmt, {
144
+ 'id': insight_id,
145
+ 'workspace_id': workspace_id,
146
+ 'title': insight.title,
147
+ 'body': insight.body,
148
+ 'severity': insight.severity,
149
+ 'score': insight.score,
150
+ 'metadata': json.dumps(insight.metadata) if insight.metadata else '{}',
151
+ 'chart_payload': json.dumps(insight.chart_payload) if insight.chart_payload else None,
152
+ 'agent': self.agent_name,
153
+ 'created_at': datetime.utcnow()
154
+ })
155
+ await db.commit()
156
+ return {'id': insight_id}
157
+
158
+ except Exception as e:
159
+ self.logger.error(f"Failed to persist insight: {e}")
160
+ # If the ai_insights table doesn't exist yet, return a generated ID
161
+ # so notification dispatch can still proceed
162
+ return {'id': str(uuid.uuid4())}
163
+
164
+ async def _dispatch_notifications(self, workspace_id: str, insight_id: str, insight: Insight):
165
+ """Send notifications to all workspace members based on their preferences"""
166
+ try:
167
+ # Get workspace members from PostgreSQL
168
+ async with AsyncSessionLocal() as db:
169
+ members = await get_workspace_members(db, workspace_id)
170
+
171
+ self.logger.info(f"Dispatching notifications to {len(members)} members")
172
+
173
+ for member in members:
174
+ user_id = member['user_id']
175
+
176
+ # Use default notification settings
177
+ # Default: email enabled, push disabled, medium threshold
178
+ default_settings = {
179
+ 'ai_insights': True,
180
+ 'email_notifications': True,
181
+ 'push_notifications': False,
182
+ 'severity_threshold': 'medium',
183
+ 'dnd_start': None,
184
+ 'dnd_end': None
185
+ }
186
+
187
+ # Check if AI insights are enabled
188
+ if not default_settings.get('ai_insights', True):
189
+ continue
190
+
191
+ # Check severity threshold
192
+ if not should_notify(insight.severity, default_settings.get('severity_threshold', 'medium')):
193
+ self.logger.info(f"Insight severity {insight.severity} below threshold for user {user_id}")
194
+ continue
195
+
196
+ # Check Do Not Disturb window
197
+ if in_dnd_window(default_settings.get('dnd_start'), default_settings.get('dnd_end')):
198
+ self.logger.info(f"User {user_id} is in DND window")
199
+ continue
200
+
201
+ # Check rate limit
202
+ if await check_rate_limit(workspace_id, user_id):
203
+ self.logger.warning(f"Rate limit exceeded for user {user_id}")
204
+ continue
205
+
206
+ # Enqueue notification
207
+ job = NotificationJob(
208
+ insight_id=insight_id,
209
+ workspace_id=workspace_id,
210
+ user_id=user_id,
211
+ channels={
212
+ 'email': default_settings.get('email_notifications', True),
213
+ 'push': default_settings.get('push_notifications', False)
214
+ },
215
+ payload={
216
+ 'title': insight.title,
217
+ 'body': insight.body,
218
+ 'chart_payload': insight.chart_payload
219
+ }
220
+ )
221
+
222
+ await enqueue_notification(job)
223
+ self.logger.info(f"Notification enqueued for user {user_id}")
224
+
225
+ except Exception as e:
226
+ self.logger.error(f"Failed to dispatch notifications: {e}", exc_info=True)
227
+
228
+ async def _log_execution(
229
+ self,
230
+ workspace_id: str,
231
+ status: str,
232
+ message: str,
233
+ metadata: Dict[str, Any] = None
234
+ ):
235
+ """Log agent execution to database"""
236
+ try:
237
+ async with AsyncSessionLocal() as db:
238
+ from sqlalchemy import text
239
+ stmt = text("""
240
+ INSERT INTO agent_logs (id, agent_name, workspace_id, status, message, metadata, created_at)
241
+ VALUES (:id, :agent_name, :workspace_id, :status, :message, :metadata, :created_at)
242
+ ON CONFLICT DO NOTHING
243
+ """)
244
+ await db.execute(stmt, {
245
+ 'id': str(uuid.uuid4()),
246
+ 'agent_name': self.agent_name,
247
+ 'workspace_id': workspace_id,
248
+ 'status': status,
249
+ 'message': message,
250
+ 'metadata': json.dumps(metadata or {}),
251
+ 'created_at': datetime.utcnow()
252
+ })
253
+ await db.commit()
254
+ except Exception as e:
255
+ # Don't let logging failures crash the agent
256
+ self.logger.error(f"Failed to log execution: {e}")
backend/agents/causal_simulator.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import logging
3
+ import pandas as pd
4
+ from typing import Dict, Any, Tuple
5
+ from core.llm import chat
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ def parse_what_if_query(query: str, columns: list) -> Dict[str, Any]:
10
+ """
11
+ Uses an LLM to extract the causal simulation parameters from a What-If query.
12
+ """
13
+ prompt = f"""You are a Causal AI Simulator.
14
+ The user is asking a "What-If" question to simulate a change in their dataset.
15
+
16
+ AVAILABLE COLUMNS IN DATASET:
17
+ {columns}
18
+
19
+ USER QUERY: "{query}"
20
+
21
+ Extract the simulation parameters.
22
+ 1. 'manipulated_variable': The exact column name from the list above that the user wants to change.
23
+ 2. 'operation': 'increase', 'decrease', 'set', or 'multiply'.
24
+ 3. 'value': The numerical value of the change (e.g. 20 for 20%).
25
+ 4. 'is_percentage': true if they said "20%", false if they said "by $20".
26
+
27
+ Output ONLY a valid JSON object:
28
+ {{
29
+ "manipulated_variable": "Column_Name",
30
+ "operation": "increase",
31
+ "value": 20,
32
+ "is_percentage": true
33
+ }}
34
+ If no simulation parameters can be parsed, return an empty JSON {{}}.
35
+ """
36
+ try:
37
+ response = chat(prompt, temperature=0.1, max_tokens=150)
38
+
39
+ # Parse JSON
40
+ response = response.strip()
41
+ if '```json' in response:
42
+ response = response.split('```json')[1].split('```')[0]
43
+ elif '```' in response:
44
+ response = response.split('```')[1].split('```')[0]
45
+
46
+ start = response.find('{')
47
+ end = response.rfind('}') + 1
48
+ if start >= 0 and end > start:
49
+ response = response[start:end]
50
+
51
+ decision = json.loads(response)
52
+ return decision
53
+ except Exception as e:
54
+ logger.error(f"Failed to parse what-if query: {e}")
55
+ return {}
56
+
57
+ def simulate_what_if(query: str, df: pd.DataFrame) -> Tuple[bool, pd.DataFrame, Dict[str, Any]]:
58
+ """
59
+ Simulates a what-if scenario on the DataFrame based on the user's query.
60
+ Returns (Success: bool, Simulated_DF: DataFrame, Simulation_Details: Dict)
61
+ """
62
+ if df is None or df.empty:
63
+ return False, df, {}
64
+
65
+ try:
66
+ columns = list(df.columns)
67
+ sim_params = parse_what_if_query(query, columns)
68
+
69
+ if not sim_params or "manipulated_variable" not in sim_params:
70
+ return False, df, {}
71
+
72
+ col = sim_params["manipulated_variable"]
73
+ op = sim_params.get("operation", "increase")
74
+ val = float(sim_params.get("value", 0))
75
+ is_pct = sim_params.get("is_percentage", False)
76
+
77
+ if col not in df.columns:
78
+ return False, df, {"error": f"Column {col} not found"}
79
+
80
+ # Create a deep copy of the dataframe for simulation
81
+ sim_df = df.copy()
82
+
83
+ # Ensure column is numeric before manipulating
84
+ if pd.api.types.is_numeric_dtype(sim_df[col]):
85
+ if op == "increase":
86
+ if is_pct:
87
+ sim_df[col] = sim_df[col] * (1 + (val / 100))
88
+ else:
89
+ sim_df[col] = sim_df[col] + val
90
+ elif op == "decrease":
91
+ if is_pct:
92
+ sim_df[col] = sim_df[col] * (1 - (val / 100))
93
+ else:
94
+ sim_df[col] = sim_df[col] - val
95
+ elif op == "multiply":
96
+ sim_df[col] = sim_df[col] * val
97
+ elif op == "set":
98
+ sim_df[col] = val
99
+
100
+ return True, sim_df, sim_params
101
+ else:
102
+ return False, df, {"error": f"Column {col} is not numeric."}
103
+
104
+ except Exception as e:
105
+ logger.error(f"Error in simulate_what_if: {e}")
106
+ return False, df, {"error": str(e)}
backend/agents/chart_gatekeeper.py ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Chart Gatekeeping - Strict validation before chart generation
2
+ """
3
+ Ensures charts are only generated when:
4
+ 1. Sufficient data exists
5
+ 2. Chart type matches data
6
+ 3. Data quality is acceptable
7
+ """
8
+
9
+ from typing import Dict, Any, Optional, Tuple
10
+ import pandas as pd
11
+
12
+
13
+ # ============================================================================
14
+ # CHART REQUIREMENTS
15
+ # ============================================================================
16
+
17
+ CHART_REQUIREMENTS = {
18
+ "bar": {
19
+ "min_data_points": 2,
20
+ "max_data_points": 50, # Too many bars = unreadable
21
+ "required_columns": ["name", "value"],
22
+ "description": "Bar chart for comparing categories"
23
+ },
24
+ "line": {
25
+ "min_data_points": 3, # Need trend
26
+ "max_data_points": 365,
27
+ "required_columns": ["x", "y"],
28
+ "description": "Line chart for trends over time"
29
+ },
30
+ "pie": {
31
+ "min_data_points": 2,
32
+ "max_data_points": 8, # More than 8 slices = confusing
33
+ "required_columns": ["label", "value"],
34
+ "description": "Pie chart for proportions"
35
+ },
36
+ "scatter": {
37
+ "min_data_points": 5,
38
+ "max_data_points": 1000,
39
+ "required_columns": ["x", "y"],
40
+ "description": "Scatter plot for correlations"
41
+ },
42
+ "prediction": {
43
+ "min_data_points": 5, # Need history for forecast
44
+ "max_data_points": 365,
45
+ "required_columns": ["date", "value"],
46
+ "description": "Forecast chart with predictions"
47
+ },
48
+ }
49
+
50
+
51
+ # ============================================================================
52
+ # GATEKEEPING FUNCTIONS
53
+ # ============================================================================
54
+
55
+ def should_render_chart(
56
+ df: Optional[pd.DataFrame],
57
+ chart_type: str,
58
+ query_type: str = None
59
+ ) -> Tuple[bool, str]:
60
+ """
61
+ Strict gatekeeping: Should we render this chart?
62
+
63
+ Returns:
64
+ Tuple[bool, str]: (should_render, reason)
65
+ """
66
+
67
+ # No data = no chart
68
+ if df is None:
69
+ return False, "No data available"
70
+
71
+ if df.empty:
72
+ return False, "Dataset is empty"
73
+
74
+ # Get requirements for chart type
75
+ requirements = CHART_REQUIREMENTS.get(chart_type.lower())
76
+ if not requirements:
77
+ return False, f"Unknown chart type: {chart_type}"
78
+
79
+ # Check minimum data points
80
+ data_points = len(df)
81
+ if data_points < requirements["min_data_points"]:
82
+ return False, f"Insufficient data: {data_points} points (need {requirements['min_data_points']} minimum)"
83
+
84
+ # Check maximum data points
85
+ if data_points > requirements["max_data_points"]:
86
+ return False, f"Too many data points: {data_points} (max {requirements['max_data_points']} for readability)"
87
+
88
+ # Check for excessive null values (>30% = unreliable)
89
+ null_ratio = df.isnull().sum().sum() / (len(df) * len(df.columns))
90
+ if null_ratio > 0.3:
91
+ return False, f"Data quality issue: {null_ratio:.0%} null values"
92
+
93
+ return True, "Chart approved"
94
+
95
+
96
+ def validate_chart_data(
97
+ df: pd.DataFrame,
98
+ chart_type: str
99
+ ) -> Tuple[bool, str, Optional[pd.DataFrame]]:
100
+ """
101
+ Validate and clean data for charting.
102
+
103
+ Returns:
104
+ Tuple[bool, str, DataFrame]: (is_valid, message, cleaned_df)
105
+ """
106
+
107
+ if df is None or df.empty:
108
+ return False, "No data to validate", None
109
+
110
+ # Make a copy to avoid modifying original
111
+ clean_df = df.copy()
112
+
113
+ # Remove rows with all nulls
114
+ clean_df = clean_df.dropna(how='all')
115
+
116
+ # Check again after cleaning
117
+ if clean_df.empty:
118
+ return False, "All data rows were empty", None
119
+
120
+ # For numeric charts, ensure we have numeric data
121
+ numeric_types = ["bar", "line", "pie", "scatter", "prediction"]
122
+ if chart_type.lower() in numeric_types:
123
+ # Find numeric columns
124
+ numeric_cols = clean_df.select_dtypes(include=['number']).columns
125
+ if len(numeric_cols) == 0:
126
+ return False, "No numeric data for chart", None
127
+
128
+ return True, "Data validated", clean_df
129
+
130
+
131
+ def get_chart_decision(
132
+ df: Optional[pd.DataFrame],
133
+ chart_type: str,
134
+ query_type: str = None,
135
+ force: bool = False
136
+ ) -> Dict[str, Any]:
137
+ """
138
+ Make a chart rendering decision with full reasoning.
139
+
140
+ Returns:
141
+ Dict with keys: should_render, reason, warnings, chart_type, data_points
142
+ """
143
+
144
+ decision = {
145
+ "should_render": False,
146
+ "reason": "",
147
+ "warnings": [],
148
+ "chart_type": chart_type,
149
+ "data_points": 0
150
+ }
151
+
152
+ if df is not None and not df.empty:
153
+ decision["data_points"] = len(df)
154
+
155
+ # Check gatekeeping
156
+ can_render, reason = should_render_chart(df, chart_type, query_type)
157
+
158
+ if not can_render and not force:
159
+ decision["reason"] = reason
160
+ return decision
161
+
162
+ # Validate data
163
+ if df is not None:
164
+ is_valid, message, _ = validate_chart_data(df, chart_type)
165
+ if not is_valid and not force:
166
+ decision["reason"] = message
167
+ return decision
168
+
169
+ # Approved
170
+ decision["should_render"] = True
171
+ decision["reason"] = "Chart approved"
172
+
173
+ # Add warnings if applicable
174
+ if decision["data_points"] < 5:
175
+ decision["warnings"].append("Limited data points - interpretation may be less reliable")
176
+
177
+ return decision
178
+
179
+
180
+ # ============================================================================
181
+ # QUERY-BASED CHART SELECTION
182
+ # ============================================================================
183
+
184
+ def suggest_chart_type(query: str, df: Optional[pd.DataFrame]) -> Optional[str]:
185
+ """
186
+ Suggest appropriate chart type based on query and data.
187
+ Returns None if no chart is appropriate.
188
+ """
189
+ query_lower = query.lower()
190
+
191
+ # Trend/time queries → line chart
192
+ if any(word in query_lower for word in ["trend", "over time", "monthly", "weekly", "daily", "growth"]):
193
+ return "line"
194
+
195
+ # Comparison queries → bar chart
196
+ if any(word in query_lower for word in ["compare", "top", "bottom", "ranking", "best", "worst"]):
197
+ return "bar"
198
+
199
+ # Distribution queries → pie chart
200
+ if any(word in query_lower for word in ["breakdown", "distribution", "percentage", "share", "proportion"]):
201
+ if df is not None and len(df) <= 8: # Pie only for small datasets
202
+ return "pie"
203
+ return "bar" # Fall back to bar for larger datasets
204
+
205
+ # Prediction queries → prediction chart
206
+ if any(word in query_lower for word in ["predict", "forecast", "future", "next"]):
207
+ return "prediction"
208
+
209
+ # Correlation queries → scatter
210
+ if any(word in query_lower for word in ["correlation", "relationship", "vs", "versus"]):
211
+ return "scatter"
212
+
213
+ # Default: no chart if not clearly needed
214
+ return None
backend/agents/chatgpt_handler.py ADDED
@@ -0,0 +1,240 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Autonomous Chat Handler - 100% LLM-Driven
3
+ ==========================================
4
+
5
+ NO hardcoded messages, NO hardcoded patterns!
6
+ Everything is generated by LLM dynamically.
7
+
8
+ This makes DataVision truly unbeatable - it adapts
9
+ to ANY data type and ANY conversation style.
10
+ """
11
+
12
+ import json
13
+ import logging
14
+ from typing import Optional, Tuple, List
15
+
16
+ from core.llm import chat
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ def handle_greeting(
22
+ query: str,
23
+ user_name: Optional[str] = None,
24
+ has_data: bool = False,
25
+ columns: List[str] = None,
26
+ domain: str = None
27
+ ) -> str:
28
+ """
29
+ Generate greeting 100% autonomously using LLM.
30
+ NO hardcoded greeting text!
31
+ """
32
+ try:
33
+ prompt = f"""You are DataVision, an AI data analyst that can analyze ANY type of data.
34
+ Generate a warm, helpful greeting response.
35
+
36
+ Context:
37
+ - User name: {user_name or "Unknown"}
38
+ - Has uploaded data: {has_data}
39
+ - Data columns: {columns or "No data yet"}
40
+ - Detected domain: {domain or "Unknown"}
41
+
42
+ Generate a personalized greeting that:
43
+ 1. Welcomes the user warmly (use their name if known)
44
+ 2. If they have data, mentions what you can analyze
45
+ 3. If no data, encourages them to upload
46
+ 4. Suggests 2-3 example questions they could ask
47
+ 5. Uses appropriate emojis
48
+
49
+ Generate the greeting (be conversational, not robotic):"""
50
+
51
+ result = chat(prompt, temperature=0.8, max_tokens=300)
52
+ return result.strip()
53
+ except Exception as e:
54
+ logger.warning(f"Greeting generation error: {e}")
55
+ # Minimal fallback
56
+ if user_name:
57
+ return f"Hello {user_name}! 👋 I'm DataVision, ready to analyze any data you throw at me!"
58
+ return "Hello! 👋 I'm DataVision. Upload any data and let's explore it together!"
59
+
60
+
61
+ def handle_off_topic(query: str, columns: List[str] = None) -> str:
62
+ """
63
+ Handle off-topic queries autonomously.
64
+ NO hardcoded responses!
65
+ """
66
+ try:
67
+ prompt = f"""The user asked something that seems off-topic for a data analysis AI.
68
+
69
+ USER QUERY: "{query}"
70
+ AVAILABLE DATA: {columns or "No data uploaded"}
71
+
72
+ Generate a polite response that:
73
+ 1. Acknowledges their question
74
+ 2. Explains you're designed for data analysis
75
+ 3. Redirects to what you CAN help with
76
+ 4. Suggests relevant data questions they could ask
77
+
78
+ Keep it friendly and helpful, not dismissive.
79
+
80
+ Response:"""
81
+
82
+ result = chat(prompt, temperature=0.7, max_tokens=200)
83
+ return result.strip()
84
+ except:
85
+ return "I'm DataVision, focused on data analysis. Try asking about your uploaded data!"
86
+
87
+
88
+ def handle_no_data() -> str:
89
+ """
90
+ Generate no-data response autonomously.
91
+ """
92
+ try:
93
+ prompt = """You are DataVision, an AI that analyzes ANY type of data.
94
+ The user hasn't uploaded any data yet.
95
+
96
+ Generate a helpful response that:
97
+ 1. Explains you need data to analyze
98
+ 2. Lists what file types you support (CSV, Excel, PDF, images)
99
+ 3. Gives examples of different domains you can handle (HR, business, scientific, IoT, etc.)
100
+ 4. Encourages them to upload and promises to help
101
+
102
+ Keep it encouraging and not overwhelming.
103
+
104
+ Response:"""
105
+
106
+ result = chat(prompt, temperature=0.7, max_tokens=250)
107
+ return result.strip()
108
+ except:
109
+ return "I need data to analyze! Upload any CSV, Excel, PDF, or image file and I'll help you explore it."
110
+
111
+
112
+ def is_greeting(query: str) -> bool:
113
+ """
114
+ Detect greeting autonomously using LLM.
115
+ """
116
+ try:
117
+ prompt = f"""Is this a greeting/hello message? Answer ONLY "yes" or "no".
118
+
119
+ MESSAGE: "{query}"
120
+
121
+ Answer:"""
122
+
123
+ result = chat(prompt, temperature=0, max_tokens=5)
124
+ return 'yes' in result.lower()
125
+ except:
126
+ # Simple fallback for performance
127
+ q = query.lower().strip()
128
+ return q in ['hi', 'hello', 'hey', 'start'] or q.startswith(('hi ', 'hello ', 'hey '))
129
+
130
+
131
+ def is_off_topic(query: str, columns: List[str] = None) -> bool:
132
+ """
133
+ Detect off-topic queries autonomously using LLM.
134
+ """
135
+ try:
136
+ prompt = f"""Is this query related to data analysis or could it be answered with data?
137
+
138
+ QUERY: "{query}"
139
+ AVAILABLE DATA COLUMNS: {columns or "Unknown"}
140
+
141
+ Answer ONLY "data_related" or "off_topic":"""
142
+
143
+ result = chat(prompt, temperature=0, max_tokens=10)
144
+ return 'off_topic' in result.lower()
145
+ except:
146
+ return False # Default to not off-topic
147
+
148
+
149
+ def detect_special_query(query: str, columns: List[str] = None) -> Tuple[str, Optional[str]]:
150
+ """
151
+ Detect special query types autonomously.
152
+ """
153
+ # Check greeting
154
+ if is_greeting(query):
155
+ return "greeting", None
156
+
157
+ # Check off-topic (only if we have data context)
158
+ if columns and is_off_topic(query, columns):
159
+ return "off_topic", handle_off_topic(query, columns)
160
+
161
+ return "analysis", None
162
+
163
+
164
+ def add_personality(response: str, query: str) -> str:
165
+ """
166
+ Add personality touches autonomously.
167
+ """
168
+ # Don't modify if already has personality
169
+ if any(emoji in response[:20] for emoji in ['👋', '🚀', '📊', '💡', '🎯']):
170
+ return response
171
+
172
+ if len(response) < 50:
173
+ return response
174
+
175
+ try:
176
+ prompt = f"""Add a brief, relevant emoji at the start of this response.
177
+ Choose based on the content (📊 for data, 📈 for trends, 🎯 for insights, etc.)
178
+
179
+ QUERY: {query[:100]}
180
+ RESPONSE START: {response[:100]}
181
+
182
+ Return ONLY the emoji (one character):"""
183
+
184
+ emoji = chat(prompt, temperature=0.3, max_tokens=5)
185
+ emoji = emoji.strip()
186
+
187
+ # Validate it's actually an emoji
188
+ if len(emoji) <= 4 and not emoji.isalnum():
189
+ return f"{emoji} {response}"
190
+ except:
191
+ pass
192
+
193
+ return response
194
+
195
+
196
+ def generate_contextual_response(
197
+ query: str,
198
+ data_context: str,
199
+ columns: List[str],
200
+ user_name: Optional[str] = None,
201
+ previous_response: Optional[str] = None
202
+ ) -> str:
203
+ """
204
+ Generate fully contextual response autonomously.
205
+ This is the core intelligence!
206
+ """
207
+ try:
208
+ context_parts = []
209
+
210
+ if user_name:
211
+ context_parts.append(f"User's name: {user_name}")
212
+
213
+ if previous_response:
214
+ context_parts.append(f"Previous response: {previous_response[:300]}")
215
+
216
+ prompt = f"""You are DataVision, an expert AI data analyst.
217
+ Analyze this query and provide a helpful, data-driven response.
218
+
219
+ QUERY: {query}
220
+
221
+ DATA CONTEXT:
222
+ Columns: {columns}
223
+ Data sample: {data_context[:1000]}
224
+
225
+ {chr(10).join(context_parts) if context_parts else ""}
226
+
227
+ Instructions:
228
+ 1. Answer based on the actual data provided
229
+ 2. Be specific with numbers and facts
230
+ 3. Format nicely with markdown
231
+ 4. If showing lists, use bullet points
232
+ 5. If data is missing, say so honestly
233
+
234
+ Response:"""
235
+
236
+ result = chat(prompt, temperature=0.3, max_tokens=800)
237
+ return result.strip()
238
+ except Exception as e:
239
+ logger.error(f"Contextual response error: {e}")
240
+ return "I encountered an issue analyzing your query. Please try rephrasing."
backend/agents/comprehensive_charts.py ADDED
@@ -0,0 +1,706 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ COMPREHENSIVE CHART SYSTEM - ALL 15+ Chart Types
3
+ ==================================================
4
+
5
+ DataVision supports ALL major visualization types!
6
+ Everything autonomous - LLM picks the right chart for the query.
7
+
8
+ CHART TYPES SUPPORTED:
9
+ 1. Bar Chart (vertical)
10
+ 2. Horizontal Bar Chart
11
+ 3. Stacked Bar Chart
12
+ 4. Line Chart
13
+ 5. Area Chart
14
+ 6. Scatter Plot
15
+ 7. Pie Chart
16
+ 8. Donut Chart
17
+ 9. Sunburst (Hierarchical)
18
+ 10. Treemap
19
+ 11. Heatmap
20
+ 12. Radar/Spider Chart
21
+ 13. Gauge Chart
22
+ 14. Waterfall Chart
23
+ 15. Funnel Chart
24
+ 16. Box Plot
25
+ 17. Histogram
26
+ 18. Candlestick (Financial)
27
+
28
+ All generated from USER'S DATA - never hardcoded!
29
+ """
30
+
31
+ import json
32
+ import logging
33
+ from typing import Dict, List, Optional, Any, Tuple
34
+ import pandas as pd
35
+ import numpy as np
36
+
37
+ from core.llm import chat
38
+
39
+ logger = logging.getLogger(__name__)
40
+
41
+
42
+ # ============================================================================
43
+ # QUERY-BASED DYNAMIC COLOR SYSTEM
44
+ # Colors are selected based on what the user is asking about
45
+ # ============================================================================
46
+
47
+ # Color palettes organized by query intent
48
+ QUERY_COLOR_PALETTES = {
49
+ # Money/Revenue related - Greens (money = green)
50
+ "sales": ['#10b981', '#059669', '#34d399', '#6ee7b7', '#a7f3d0', '#22c55e', '#4ade80', '#86efac'],
51
+ "revenue": ['#10b981', '#059669', '#34d399', '#6ee7b7', '#a7f3d0', '#22c55e', '#4ade80', '#86efac'],
52
+ "income": ['#10b981', '#059669', '#34d399', '#6ee7b7', '#a7f3d0', '#22c55e', '#4ade80', '#86efac'],
53
+
54
+ # Profit/Growth - Teal (positive trends)
55
+ "profit": ['#14b8a6', '#0d9488', '#2dd4bf', '#5eead4', '#99f6e4', '#0f766e', '#0891b2', '#06b6d4'],
56
+ "growth": ['#14b8a6', '#0d9488', '#2dd4bf', '#5eead4', '#99f6e4', '#0f766e', '#0891b2', '#06b6d4'],
57
+ "margin": ['#14b8a6', '#0d9488', '#2dd4bf', '#5eead4', '#99f6e4', '#0f766e', '#0891b2', '#06b6d4'],
58
+
59
+ # Loss/Decline - Reds (warning)
60
+ "loss": ['#ef4444', '#dc2626', '#f87171', '#fca5a5', '#fecaca', '#b91c1c', '#f43f5e', '#fb7185'],
61
+ "decline": ['#ef4444', '#dc2626', '#f87171', '#fca5a5', '#fecaca', '#b91c1c', '#f43f5e', '#fb7185'],
62
+ "risk": ['#ef4444', '#dc2626', '#f87171', '#fca5a5', '#fecaca', '#b91c1c', '#f43f5e', '#fb7185'],
63
+ "cost": ['#ef4444', '#dc2626', '#f87171', '#fca5a5', '#fecaca', '#b91c1c', '#f43f5e', '#fb7185'],
64
+
65
+ # Performance/Rating - Purple (premium feel)
66
+ "rating": ['#8b5cf6', '#7c3aed', '#a78bfa', '#c4b5fd', '#ddd6fe', '#6d28d9', '#a855f7', '#c084fc'],
67
+ "performance": ['#8b5cf6', '#7c3aed', '#a78bfa', '#c4b5fd', '#ddd6fe', '#6d28d9', '#a855f7', '#c084fc'],
68
+ "quality": ['#8b5cf6', '#7c3aed', '#a78bfa', '#c4b5fd', '#ddd6fe', '#6d28d9', '#a855f7', '#c084fc'],
69
+
70
+ # Time/Trend - Blue (timeline feel)
71
+ "time": ['#3b82f6', '#2563eb', '#60a5fa', '#93c5fd', '#bfdbfe', '#1d4ed8', '#6366f1', '#818cf8'],
72
+ "trend": ['#3b82f6', '#2563eb', '#60a5fa', '#93c5fd', '#bfdbfe', '#1d4ed8', '#6366f1', '#818cf8'],
73
+ "month": ['#3b82f6', '#2563eb', '#60a5fa', '#93c5fd', '#bfdbfe', '#1d4ed8', '#6366f1', '#818cf8'],
74
+ "year": ['#3b82f6', '#2563eb', '#60a5fa', '#93c5fd', '#bfdbfe', '#1d4ed8', '#6366f1', '#818cf8'],
75
+
76
+ # Categories/Segments - Orange (variety)
77
+ "category": ['#f97316', '#ea580c', '#fb923c', '#fdba74', '#fed7aa', '#c2410c', '#f59e0b', '#fbbf24'],
78
+ "segment": ['#f97316', '#ea580c', '#fb923c', '#fdba74', '#fed7aa', '#c2410c', '#f59e0b', '#fbbf24'],
79
+ "product": ['#f97316', '#ea580c', '#fb923c', '#fdba74', '#fed7aa', '#c2410c', '#f59e0b', '#fbbf24'],
80
+ "region": ['#f97316', '#ea580c', '#fb923c', '#fdba74', '#fed7aa', '#c2410c', '#f59e0b', '#fbbf24'],
81
+
82
+ # Customers - Pink (people/relationship)
83
+ "customer": ['#ec4899', '#db2777', '#f472b6', '#f9a8d4', '#fbcfe8', '#be185d', '#d946ef', '#e879f9'],
84
+ "user": ['#ec4899', '#db2777', '#f472b6', '#f9a8d4', '#fbcfe8', '#be185d', '#d946ef', '#e879f9'],
85
+ "client": ['#ec4899', '#db2777', '#f472b6', '#f9a8d4', '#fbcfe8', '#be185d', '#d946ef', '#e879f9'],
86
+
87
+ # Quantity/Volume - Cyan
88
+ "quantity": ['#06b6d4', '#0891b2', '#22d3ee', '#67e8f9', '#a5f3fc', '#0e7490', '#14b8a6', '#2dd4bf'],
89
+ "volume": ['#06b6d4', '#0891b2', '#22d3ee', '#67e8f9', '#a5f3fc', '#0e7490', '#14b8a6', '#2dd4bf'],
90
+ "count": ['#06b6d4', '#0891b2', '#22d3ee', '#67e8f9', '#a5f3fc', '#0e7490', '#14b8a6', '#2dd4bf'],
91
+ }
92
+
93
+ # Default palette (teal theme)
94
+ DEFAULT_COLORS = ['#14b8a6', '#0d9488', '#0f766e', '#06b6d4', '#0891b2', '#22d3ee', '#10b981', '#22c55e']
95
+
96
+ # Current query context for color selection
97
+ _current_query = ""
98
+
99
+ def set_query_context(query: str):
100
+ """Set the current query for color selection"""
101
+ global _current_query
102
+ _current_query = query.lower()
103
+
104
+ def get_colors_for_query(query: str = None) -> List[str]:
105
+ """Get colors based on query intent"""
106
+ q = (query or _current_query).lower()
107
+
108
+ # Check each keyword in the query
109
+ for keyword, colors in QUERY_COLOR_PALETTES.items():
110
+ if keyword in q:
111
+ return colors
112
+
113
+ # Default to teal theme
114
+ return DEFAULT_COLORS
115
+
116
+ def get_current_colors() -> List[str]:
117
+ """Get current color palette based on query context"""
118
+ return get_colors_for_query(_current_query)
119
+
120
+
121
+ # ============================================================================
122
+ # ALL CHART GENERATORS - Enhanced with Interactivity
123
+ # ============================================================================
124
+
125
+ # Common interactive layout settings
126
+ INTERACTIVE_LAYOUT = {
127
+ "paper_bgcolor": "rgba(0,0,0,0)",
128
+ "plot_bgcolor": "rgba(0,0,0,0)",
129
+ "font": {"color": "#9ca3af", "family": "Inter, system-ui, sans-serif"},
130
+ "hoverlabel": {
131
+ "bgcolor": "#1f2937",
132
+ "bordercolor": "#374151",
133
+ "font": {"color": "#fff", "size": 13}
134
+ },
135
+ "dragmode": "zoom",
136
+ "hovermode": "closest"
137
+ }
138
+
139
+ def generate_bar_chart(labels: List, values: List, title: str, horizontal: bool = False) -> Dict:
140
+ """Generate interactive bar chart with animations and hover effects"""
141
+ colors = get_current_colors()
142
+
143
+ # Create gradient colors for each bar
144
+ num_bars = len(labels)
145
+ bar_colors = colors[:num_bars] if num_bars <= len(colors) else (colors * ((num_bars // len(colors)) + 1))[:num_bars]
146
+
147
+ # Calculate percentage of max for context
148
+ max_val = max(values) if values else 1
149
+
150
+ if horizontal:
151
+ return {
152
+ "data": [{
153
+ "type": "bar",
154
+ "y": [str(l) for l in labels],
155
+ "x": values,
156
+ "orientation": "h",
157
+ "marker": {
158
+ "color": bar_colors,
159
+ "line": {"color": "rgba(255,255,255,0.2)", "width": 1}
160
+ },
161
+ "text": [f"{v:,.0f}" if isinstance(v, (int, float)) else str(v) for v in values],
162
+ "textposition": "outside",
163
+ "textfont": {"color": "#9ca3af"},
164
+ "hovertemplate": "<b>%{y}</b><br>Value: %{x:,.0f}<br>%{customdata:.1f}% of max<extra></extra>",
165
+ "customdata": [v/max_val*100 for v in values]
166
+ }],
167
+ "layout": {
168
+ **INTERACTIVE_LAYOUT,
169
+ "title": {"text": f"📊 {title}", "font": {"color": "#fff", "size": 18}},
170
+ "xaxis": {"gridcolor": "#374151", "tickformat": ",.0f", "zeroline": False},
171
+ "yaxis": {"gridcolor": "#374151", "automargin": True},
172
+ "margin": {"l": 120, "r": 60, "t": 70, "b": 50},
173
+ "bargap": 0.3
174
+ }
175
+ }
176
+ else:
177
+ return {
178
+ "data": [{
179
+ "type": "bar",
180
+ "x": [str(l) for l in labels],
181
+ "y": values,
182
+ "marker": {
183
+ "color": bar_colors,
184
+ "line": {"color": "rgba(255,255,255,0.2)", "width": 1}
185
+ },
186
+ "text": [f"{v:,.0f}" if isinstance(v, (int, float)) else str(v) for v in values],
187
+ "textposition": "outside",
188
+ "textfont": {"color": "#9ca3af"},
189
+ "hovertemplate": "<b>%{x}</b><br>Value: %{y:,.0f}<br>%{customdata:.1f}% of max<extra></extra>",
190
+ "customdata": [v/max_val*100 for v in values]
191
+ }],
192
+ "layout": {
193
+ **INTERACTIVE_LAYOUT,
194
+ "title": {"text": f"📊 {title}", "font": {"color": "#fff", "size": 18}},
195
+ "xaxis": {"gridcolor": "#374151", "tickangle": -45, "automargin": True},
196
+ "yaxis": {"gridcolor": "#374151", "tickformat": ",.0f", "zeroline": False},
197
+ "margin": {"l": 70, "r": 40, "t": 70, "b": 100},
198
+ "bargap": 0.25
199
+ }
200
+ }
201
+
202
+
203
+ def generate_stacked_bar(df: pd.DataFrame, x_col: str, y_col: str, stack_col: str, title: str) -> Dict:
204
+ """Generate stacked bar chart"""
205
+ traces = []
206
+ colors = ["#14b8a6", "#6366f1", "#f59e0b", "#ef4444", "#8b5cf6", "#06b6d4"]
207
+
208
+ for i, category in enumerate(df[stack_col].unique()[:6]):
209
+ subset = df[df[stack_col] == category]
210
+ grouped = subset.groupby(x_col)[y_col].sum()
211
+ traces.append({
212
+ "type": "bar",
213
+ "name": str(category),
214
+ "x": grouped.index.tolist(),
215
+ "y": grouped.values.tolist(),
216
+ "marker": {"color": colors[i % len(colors)]}
217
+ })
218
+
219
+ return {
220
+ "data": traces,
221
+ "layout": {
222
+ "title": {"text": title, "font": {"color": "#fff"}},
223
+ "barmode": "stack",
224
+ "paper_bgcolor": "rgba(0,0,0,0)",
225
+ "plot_bgcolor": "rgba(0,0,0,0)",
226
+ "font": {"color": "#9ca3af"}
227
+ }
228
+ }
229
+
230
+
231
+ def generate_line_chart(labels: List, values: List, title: str, fill: bool = False) -> Dict:
232
+ """Generate interactive line chart with animations and hover effects"""
233
+ colors = get_current_colors()
234
+ line_color = colors[0]
235
+
236
+ # Calculate stats for hover
237
+ avg_val = sum(values) / len(values) if values else 0
238
+
239
+ return {
240
+ "data": [{
241
+ "type": "scatter",
242
+ "mode": "lines+markers",
243
+ "x": [str(l) for l in labels],
244
+ "y": values,
245
+ "line": {"color": line_color, "width": 3, "shape": "spline"},
246
+ "marker": {"size": 10, "color": line_color, "line": {"color": "#fff", "width": 2}},
247
+ "fill": "tozeroy" if fill else None,
248
+ "fillcolor": f"rgba({int(line_color[1:3], 16)}, {int(line_color[3:5], 16)}, {int(line_color[5:7], 16)}, 0.2)" if fill else None,
249
+ "hovertemplate": "<b>%{x}</b><br>Value: %{y:,.0f}<br>Avg: " + f"{avg_val:,.0f}" + "<extra></extra>"
250
+ }],
251
+ "layout": {
252
+ **INTERACTIVE_LAYOUT,
253
+ "title": {"text": f"📈 {title}", "font": {"color": "#fff", "size": 18}},
254
+ "xaxis": {"gridcolor": "#374151", "tickangle": -45, "showgrid": True},
255
+ "yaxis": {"gridcolor": "#374151", "tickformat": ",.0f", "zeroline": False},
256
+ "margin": {"l": 70, "r": 40, "t": 70, "b": 100}
257
+ }
258
+ }
259
+
260
+
261
+ def generate_scatter_chart(x_values: List, y_values: List, title: str, labels: List = None) -> Dict:
262
+ """Generate interactive scatter plot with size scaling and gradient"""
263
+ colors = get_current_colors()
264
+
265
+ return {
266
+ "data": [{
267
+ "type": "scatter",
268
+ "mode": "markers",
269
+ "x": x_values,
270
+ "y": y_values,
271
+ "text": labels if labels else [f"({x}, {y})" for x, y in zip(x_values, y_values)],
272
+ "marker": {
273
+ "size": 14,
274
+ "color": y_values,
275
+ "colorscale": [[0, colors[0]], [0.5, colors[2]], [1, colors[4]]],
276
+ "showscale": True,
277
+ "colorbar": {"title": "Value", "tickfont": {"color": "#9ca3af"}},
278
+ "line": {"color": "#fff", "width": 1}
279
+ },
280
+ "hovertemplate": "<b>%{text}</b><br>X: %{x:,.0f}<br>Y: %{y:,.0f}<extra></extra>"
281
+ }],
282
+ "layout": {
283
+ **INTERACTIVE_LAYOUT,
284
+ "title": {"text": f"🔵 {title}", "font": {"color": "#fff", "size": 18}},
285
+ "xaxis": {"gridcolor": "#374151", "tickformat": ",.0f", "zeroline": False},
286
+ "yaxis": {"gridcolor": "#374151", "tickformat": ",.0f", "zeroline": False},
287
+ "margin": {"l": 70, "r": 100, "t": 70, "b": 60}
288
+ }
289
+ }
290
+
291
+
292
+ def generate_pie_chart(labels: List, values: List, title: str, donut: bool = False) -> Dict:
293
+ """Generate pie chart (or donut if donut=True) with dynamic colors"""
294
+ colors = get_current_colors()
295
+
296
+ return {
297
+ "data": [{
298
+ "type": "pie",
299
+ "labels": [str(l) for l in labels],
300
+ "values": values,
301
+ "hole": 0.4 if donut else 0,
302
+ "marker": {"colors": colors},
303
+ "textinfo": "label+percent",
304
+ "textfont": {"color": "#fff"},
305
+ "hovertemplate": "%{label}: %{value:,.0f}<br>%{percent}<extra></extra>"
306
+ }],
307
+ "layout": {
308
+ "title": {"text": title, "font": {"color": "#fff", "size": 16}},
309
+ "paper_bgcolor": "rgba(0,0,0,0)",
310
+ "font": {"color": "#9ca3af"},
311
+ "showlegend": True,
312
+ "legend": {"font": {"color": "#9ca3af"}}
313
+ }
314
+ }
315
+
316
+
317
+ def generate_sunburst(labels: List, parents: List, values: List, title: str) -> Dict:
318
+ """Generate sunburst/hierarchical chart"""
319
+ return {
320
+ "data": [{
321
+ "type": "sunburst",
322
+ "labels": labels,
323
+ "parents": parents,
324
+ "values": values,
325
+ "branchvalues": "total",
326
+ "marker": {"colorscale": "Teal"}
327
+ }],
328
+ "layout": {
329
+ "title": {"text": title, "font": {"color": "#fff"}},
330
+ "paper_bgcolor": "rgba(0,0,0,0)",
331
+ "font": {"color": "#9ca3af"}
332
+ }
333
+ }
334
+
335
+
336
+ def generate_treemap(labels: List, parents: List, values: List, title: str) -> Dict:
337
+ """Generate treemap chart"""
338
+ return {
339
+ "data": [{
340
+ "type": "treemap",
341
+ "labels": labels,
342
+ "parents": parents,
343
+ "values": values,
344
+ "marker": {"colorscale": "Teal"},
345
+ "textinfo": "label+value"
346
+ }],
347
+ "layout": {
348
+ "title": {"text": title, "font": {"color": "#fff"}},
349
+ "paper_bgcolor": "rgba(0,0,0,0)",
350
+ "font": {"color": "#9ca3af"}
351
+ }
352
+ }
353
+
354
+
355
+ def generate_heatmap(z_values: List, x_labels: List, y_labels: List, title: str) -> Dict:
356
+ """Generate heatmap"""
357
+ return {
358
+ "data": [{
359
+ "type": "heatmap",
360
+ "z": z_values,
361
+ "x": x_labels,
362
+ "y": y_labels,
363
+ "colorscale": "Viridis"
364
+ }],
365
+ "layout": {
366
+ "title": {"text": title, "font": {"color": "#fff"}},
367
+ "paper_bgcolor": "rgba(0,0,0,0)",
368
+ "font": {"color": "#9ca3af"}
369
+ }
370
+ }
371
+
372
+
373
+ def generate_radar_chart(categories: List, values: List, title: str) -> Dict:
374
+ """Generate radar/spider chart"""
375
+ return {
376
+ "data": [{
377
+ "type": "scatterpolar",
378
+ "r": values + [values[0]], # Close the shape
379
+ "theta": categories + [categories[0]],
380
+ "fill": "toself",
381
+ "line": {"color": "#14b8a6"},
382
+ "fillcolor": "rgba(20, 184, 166, 0.3)"
383
+ }],
384
+ "layout": {
385
+ "title": {"text": title, "font": {"color": "#fff"}},
386
+ "paper_bgcolor": "rgba(0,0,0,0)",
387
+ "font": {"color": "#9ca3af"},
388
+ "polar": {"bgcolor": "rgba(0,0,0,0)"}
389
+ }
390
+ }
391
+
392
+
393
+ def generate_gauge_chart(value: float, max_value: float, title: str) -> Dict:
394
+ """Generate gauge/meter chart"""
395
+ return {
396
+ "data": [{
397
+ "type": "indicator",
398
+ "mode": "gauge+number+delta",
399
+ "value": value,
400
+ "title": {"text": title, "font": {"color": "#fff"}},
401
+ "gauge": {
402
+ "axis": {"range": [0, max_value]},
403
+ "bar": {"color": "#14b8a6"},
404
+ "steps": [
405
+ {"range": [0, max_value*0.5], "color": "#374151"},
406
+ {"range": [max_value*0.5, max_value*0.75], "color": "#4b5563"},
407
+ {"range": [max_value*0.75, max_value], "color": "#6b7280"}
408
+ ],
409
+ "threshold": {"value": value, "thickness": 0.75, "line": {"color": "#14b8a6", "width": 4}}
410
+ }
411
+ }],
412
+ "layout": {
413
+ "paper_bgcolor": "rgba(0,0,0,0)",
414
+ "font": {"color": "#9ca3af"}
415
+ }
416
+ }
417
+
418
+
419
+ def generate_waterfall_chart(labels: List, values: List, title: str) -> Dict:
420
+ """Generate waterfall chart"""
421
+ return {
422
+ "data": [{
423
+ "type": "waterfall",
424
+ "x": labels,
425
+ "y": values,
426
+ "connector": {"line": {"color": "#6b7280"}},
427
+ "increasing": {"marker": {"color": "#10b981"}},
428
+ "decreasing": {"marker": {"color": "#ef4444"}},
429
+ "totals": {"marker": {"color": "#6366f1"}}
430
+ }],
431
+ "layout": {
432
+ "title": {"text": title, "font": {"color": "#fff"}},
433
+ "paper_bgcolor": "rgba(0,0,0,0)",
434
+ "plot_bgcolor": "rgba(0,0,0,0)",
435
+ "font": {"color": "#9ca3af"}
436
+ }
437
+ }
438
+
439
+
440
+ def generate_funnel_chart(labels: List, values: List, title: str) -> Dict:
441
+ """Generate funnel chart"""
442
+ return {
443
+ "data": [{
444
+ "type": "funnel",
445
+ "y": labels,
446
+ "x": values,
447
+ "textinfo": "value+percent initial",
448
+ "marker": {"color": ["#14b8a6", "#6366f1", "#f59e0b", "#ef4444", "#8b5cf6"]}
449
+ }],
450
+ "layout": {
451
+ "title": {"text": title, "font": {"color": "#fff"}},
452
+ "paper_bgcolor": "rgba(0,0,0,0)",
453
+ "font": {"color": "#9ca3af"}
454
+ }
455
+ }
456
+
457
+
458
+ def generate_box_plot(df: pd.DataFrame, column: str, title: str) -> Dict:
459
+ """Generate box plot for distribution"""
460
+ return {
461
+ "data": [{
462
+ "type": "box",
463
+ "y": df[column].dropna().tolist(),
464
+ "name": column,
465
+ "marker": {"color": "#14b8a6"},
466
+ "boxpoints": "outliers"
467
+ }],
468
+ "layout": {
469
+ "title": {"text": title, "font": {"color": "#fff"}},
470
+ "paper_bgcolor": "rgba(0,0,0,0)",
471
+ "plot_bgcolor": "rgba(0,0,0,0)",
472
+ "font": {"color": "#9ca3af"}
473
+ }
474
+ }
475
+
476
+
477
+ def generate_histogram(values: List, title: str, bins: int = 20) -> Dict:
478
+ """Generate histogram"""
479
+ return {
480
+ "data": [{
481
+ "type": "histogram",
482
+ "x": values,
483
+ "nbinsx": bins,
484
+ "marker": {"color": "#14b8a6"}
485
+ }],
486
+ "layout": {
487
+ "title": {"text": title, "font": {"color": "#fff"}},
488
+ "paper_bgcolor": "rgba(0,0,0,0)",
489
+ "plot_bgcolor": "rgba(0,0,0,0)",
490
+ "font": {"color": "#9ca3af"},
491
+ "xaxis": {"gridcolor": "#374151"},
492
+ "yaxis": {"gridcolor": "#374151"}
493
+ }
494
+ }
495
+
496
+
497
+ def generate_candlestick(df: pd.DataFrame, date_col: str, open_col: str, high_col: str, low_col: str, close_col: str, title: str) -> Dict:
498
+ """Generate candlestick chart for financial data"""
499
+ return {
500
+ "data": [{
501
+ "type": "candlestick",
502
+ "x": df[date_col].tolist(),
503
+ "open": df[open_col].tolist(),
504
+ "high": df[high_col].tolist(),
505
+ "low": df[low_col].tolist(),
506
+ "close": df[close_col].tolist(),
507
+ "increasing": {"line": {"color": "#10b981"}},
508
+ "decreasing": {"line": {"color": "#ef4444"}}
509
+ }],
510
+ "layout": {
511
+ "title": {"text": title, "font": {"color": "#fff"}},
512
+ "paper_bgcolor": "rgba(0,0,0,0)",
513
+ "plot_bgcolor": "rgba(0,0,0,0)",
514
+ "font": {"color": "#9ca3af"},
515
+ "xaxis": {"rangeslider": {"visible": False}}
516
+ }
517
+ }
518
+
519
+
520
+ # ============================================================================
521
+ # AUTONOMOUS CHART SELECTION - LLM decides everything
522
+ # ============================================================================
523
+
524
+ def get_chart_type_from_query(query: str, columns: List[str], sample_data: str = "") -> Dict:
525
+ """
526
+ Use chart intent detection FIRST, then LLM as fallback.
527
+ This ensures user gets exactly what they ask for.
528
+ """
529
+ try:
530
+ # STEP 1: Check for explicit user intent first (faster, no LLM call)
531
+ from core.chart_selector import ChartSelector
532
+ selector = ChartSelector()
533
+ detected_charts = selector.detect_chart_intent(query)
534
+
535
+ if detected_charts:
536
+ # User explicitly asked for a specific chart type
537
+ preferred_chart = detected_charts[0] # First match is most specific
538
+ x_col = columns[0] if columns else "category"
539
+ y_col = columns[-1] if len(columns) > 1 else columns[0] if columns else "value"
540
+
541
+ return {
542
+ "chart_type": preferred_chart,
543
+ "x_column": x_col,
544
+ "y_column": y_col,
545
+ "title": f"{y_col} by {x_col}".replace("_", " ").title(),
546
+ "reason": f"User explicitly requested {preferred_chart} visualization"
547
+ }
548
+
549
+ # STEP 2: LLM fallback for ambiguous queries
550
+ prompt = f"""Analyze this query and data to recommend the BEST visualization.
551
+
552
+ QUERY: "{query}"
553
+ COLUMNS: {columns}
554
+ SAMPLE: {sample_data[:400]}
555
+
556
+ AVAILABLE CHART TYPES:
557
+ 1. bar - Comparing categories
558
+ 2. horizontal_bar - Long category names
559
+ 3. stacked_bar - Multiple series comparison
560
+ 4. line - Trends over time
561
+ 5. area - Volume/cumulative trends
562
+ 6. scatter - Correlation/relationship
563
+ 7. pie - Part of whole (few categories)
564
+ 8. donut - Part of whole (with center)
565
+ 9. sunburst - Hierarchical data
566
+ 10. treemap - Hierarchical with sizes
567
+ 11. heatmap - Two-dimensional correlation
568
+ 12. radar - Multi-variable comparison
569
+ 13. gauge - Single KPI value
570
+ 14. waterfall - Incremental changes
571
+ 15. funnel - Stage progression
572
+ 16. box - Distribution/outliers
573
+ 17. histogram - Value distribution
574
+ 18. candlestick - Financial OHLC
575
+
576
+ Return JSON:
577
+ {{"chart_type": "best_type", "x_column": "col", "y_column": "col", "title": "title", "reason": "why"}}
578
+
579
+ JSON:"""
580
+
581
+ result = chat(prompt, temperature=0.1, max_tokens=200)
582
+
583
+ # Parse
584
+ result = result.strip()
585
+ if '```' in result:
586
+ result = result.split('```')[1]
587
+ if result.startswith('json'):
588
+ result = result[4:]
589
+
590
+ start = result.find('{')
591
+ end = result.rfind('}') + 1
592
+ if start >= 0 and end > start:
593
+ result = result[start:end]
594
+
595
+ return json.loads(result)
596
+
597
+ except Exception as e:
598
+ return {"chart_type": "bar", "reason": str(e)}
599
+
600
+
601
+ def generate_autonomous_visualization(query: str, df: pd.DataFrame) -> Optional[str]:
602
+ """Generate ANY chart type autonomously from user data with query-based colors"""
603
+ if df is None or df.empty:
604
+ return None
605
+
606
+ try:
607
+ # Set query context for dynamic color selection
608
+ set_query_context(query)
609
+
610
+ columns = list(df.columns)
611
+ sample = df.head(5).to_string()
612
+
613
+ # Get LLM recommendation
614
+ recommendation = get_chart_type_from_query(query, columns, sample)
615
+ chart_type = recommendation.get("chart_type", "bar")
616
+ x_col = recommendation.get("x_column", columns[0])
617
+ y_col = recommendation.get("y_column", columns[-1] if len(columns) > 1 else columns[0])
618
+ title = recommendation.get("title", f"{y_col} by {x_col}")
619
+
620
+ # Validate columns
621
+ if x_col not in columns:
622
+ x_col = columns[0]
623
+ if y_col not in columns:
624
+ y_col = columns[-1] if len(columns) > 1 else columns[0]
625
+
626
+ # Prepare data
627
+ if df[y_col].dtype in ['int64', 'float64']:
628
+ grouped = df.groupby(x_col)[y_col].sum().head(15)
629
+ labels = grouped.index.tolist()
630
+ values = grouped.values.tolist()
631
+ else:
632
+ counts = df[x_col].value_counts().head(15)
633
+ labels = counts.index.tolist()
634
+ values = counts.values.tolist()
635
+
636
+ # Generate chart based on type
637
+ if chart_type == "horizontal_bar":
638
+ chart_json = generate_bar_chart(labels, values, title, horizontal=True)
639
+ elif chart_type == "line":
640
+ chart_json = generate_line_chart(labels, values, title)
641
+ elif chart_type == "area":
642
+ chart_json = generate_line_chart(labels, values, title, fill=True)
643
+ elif chart_type == "scatter":
644
+ chart_json = generate_scatter_chart(labels, values, title)
645
+ elif chart_type == "pie":
646
+ chart_json = generate_pie_chart(labels[:8], values[:8], title)
647
+ elif chart_type == "donut":
648
+ chart_json = generate_pie_chart(labels[:8], values[:8], title, donut=True)
649
+ elif chart_type == "funnel":
650
+ chart_json = generate_funnel_chart(labels[:5], values[:5], title)
651
+ elif chart_type == "radar":
652
+ chart_json = generate_radar_chart(labels[:8], values[:8], title)
653
+ elif chart_type == "histogram":
654
+ if df[y_col].dtype in ['int64', 'float64']:
655
+ chart_json = generate_histogram(df[y_col].dropna().tolist(), title)
656
+ else:
657
+ chart_json = generate_bar_chart(labels, values, title)
658
+ elif chart_type == "box":
659
+ if df[y_col].dtype in ['int64', 'float64']:
660
+ chart_json = generate_box_plot(df, y_col, title)
661
+ else:
662
+ chart_json = generate_bar_chart(labels, values, title)
663
+ elif chart_type == "gauge":
664
+ total = sum(values)
665
+ chart_json = generate_gauge_chart(total, total * 1.5, title)
666
+ elif chart_type == "waterfall":
667
+ chart_json = generate_waterfall_chart(labels[:10], values[:10], title)
668
+ else: # Default to bar
669
+ chart_json = generate_bar_chart(labels, values, title)
670
+
671
+ return f"\n\n```plotly_chart\n{json.dumps(chart_json)}\n```"
672
+
673
+ except Exception as e:
674
+ logger.error(f"Autonomous visualization error: {e}")
675
+ return None
676
+
677
+
678
+ # ============================================================================
679
+ # CHART TYPES REGISTRY
680
+ # ============================================================================
681
+
682
+ SUPPORTED_CHART_TYPES = {
683
+ "bar": "📊 Bar Chart - Compare categories",
684
+ "horizontal_bar": "📊 Horizontal Bar - Long category names",
685
+ "stacked_bar": "📊 Stacked Bar - Multiple series",
686
+ "line": "📈 Line Chart - Trends over time",
687
+ "area": "📈 Area Chart - Volume trends",
688
+ "scatter": "🔵 Scatter Plot - Correlations",
689
+ "pie": "🥧 Pie Chart - Part of whole",
690
+ "donut": "🍩 Donut Chart - Part of whole",
691
+ "sunburst": "☀️ Sunburst - Hierarchical data",
692
+ "treemap": "🗺️ Treemap - Hierarchical sizes",
693
+ "heatmap": "🔥 Heatmap - 2D correlation",
694
+ "radar": "🕸️ Radar Chart - Multi-variable",
695
+ "gauge": "⏱️ Gauge Chart - Single KPI",
696
+ "waterfall": "💧 Waterfall - Incremental changes",
697
+ "funnel": "🔻 Funnel Chart - Stage progression",
698
+ "box": "📦 Box Plot - Distribution",
699
+ "histogram": "📊 Histogram - Value distribution",
700
+ "candlestick": "📈 Candlestick - Financial OHLC"
701
+ }
702
+
703
+
704
+ def get_supported_charts() -> List[str]:
705
+ """Return list of all supported chart types"""
706
+ return list(SUPPORTED_CHART_TYPES.keys())
backend/agents/confidence_scorer.py ADDED
@@ -0,0 +1,469 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Enterprise Confidence Scoring System
2
+ """
3
+ Advanced confidence scoring for AI responses.
4
+
5
+ Features:
6
+ - RAG similarity-based scoring
7
+ - Graph relevance scoring
8
+ - Hybrid fusion confidence
9
+ - Answer quality metrics
10
+ - Source attribution confidence
11
+ """
12
+
13
+ from dataclasses import dataclass
14
+ from typing import List, Dict, Optional, Tuple
15
+ import numpy as np
16
+ from enum import Enum
17
+
18
+
19
+ class ConfidenceLevel(Enum):
20
+ """Confidence level categories"""
21
+ VERY_HIGH = "very_high" # 0.9+
22
+ HIGH = "high" # 0.75-0.9
23
+ MODERATE = "moderate" # 0.5-0.75
24
+ LOW = "low" # 0.25-0.5
25
+ VERY_LOW = "very_low" # <0.25
26
+
27
+
28
+ @dataclass
29
+ class SourceScore:
30
+ """Score for individual source"""
31
+ source_name: str
32
+ relevance_score: float
33
+ coverage_score: float
34
+ recency_score: float
35
+ combined_score: float
36
+
37
+
38
+ @dataclass
39
+ class RAGConfidence:
40
+ """RAG-specific confidence metrics"""
41
+ similarity_scores: List[float]
42
+ average_similarity: float
43
+ top_similarity: float
44
+ coverage_ratio: float # How much of query is covered
45
+ source_diversity: float # Variety of sources used
46
+ overall_score: float
47
+
48
+
49
+ @dataclass
50
+ class GraphConfidence:
51
+ """GraphRAG-specific confidence metrics"""
52
+ path_relevance: float
53
+ hop_count: int
54
+ node_importance: float # PageRank-based
55
+ edge_strength: float
56
+ reasoning_depth: float
57
+ overall_score: float
58
+
59
+
60
+ @dataclass
61
+ class HybridConfidence:
62
+ """Hybrid mode confidence combining RAG + Graph"""
63
+ rag_confidence: RAGConfidence
64
+ graph_confidence: GraphConfidence
65
+ fusion_weight_rag: float
66
+ fusion_weight_graph: float
67
+ agreement_score: float # How much RAG and Graph agree
68
+ overall_score: float
69
+
70
+
71
+ @dataclass
72
+ class AnswerConfidence:
73
+ """Complete answer confidence assessment"""
74
+ mode: str
75
+ overall_score: float
76
+ confidence_level: ConfidenceLevel
77
+ rag_metrics: Optional[RAGConfidence]
78
+ graph_metrics: Optional[GraphConfidence]
79
+ hybrid_metrics: Optional[HybridConfidence]
80
+ source_scores: List[SourceScore]
81
+ explanation: str
82
+ should_flag_uncertainty: bool
83
+
84
+
85
+ class EnterpriseConfidenceScorer:
86
+ """
87
+ Enterprise-grade confidence scoring system.
88
+
89
+ Provides calibrated confidence scores for:
90
+ - RAG retrieval quality
91
+ - Graph reasoning strength
92
+ - Hybrid fusion reliability
93
+ - Overall answer quality
94
+ """
95
+
96
+ # Thresholds for confidence levels
97
+ THRESHOLDS = {
98
+ ConfidenceLevel.VERY_HIGH: 0.9,
99
+ ConfidenceLevel.HIGH: 0.75,
100
+ ConfidenceLevel.MODERATE: 0.5,
101
+ ConfidenceLevel.LOW: 0.25,
102
+ ConfidenceLevel.VERY_LOW: 0.0
103
+ }
104
+
105
+ def __init__(self):
106
+ self.calibration_factor = 1.0 # Can be tuned based on feedback
107
+
108
+ def score_rag_confidence(
109
+ self,
110
+ similarity_scores: List[float],
111
+ sources: List[str],
112
+ query_terms: int,
113
+ matched_terms: int
114
+ ) -> RAGConfidence:
115
+ """
116
+ Calculate RAG retrieval confidence.
117
+
118
+ Args:
119
+ similarity_scores: Cosine similarity scores from retrieval
120
+ sources: List of source documents
121
+ query_terms: Number of terms in query
122
+ matched_terms: Number of terms found in retrieved docs
123
+ """
124
+ if not similarity_scores:
125
+ return RAGConfidence(
126
+ similarity_scores=[],
127
+ average_similarity=0.0,
128
+ top_similarity=0.0,
129
+ coverage_ratio=0.0,
130
+ source_diversity=0.0,
131
+ overall_score=0.0
132
+ )
133
+
134
+ # Normalize similarity scores (FAISS L2 to 0-1)
135
+ normalized_scores = [self._normalize_similarity(s) for s in similarity_scores]
136
+
137
+ avg_sim = np.mean(normalized_scores)
138
+ top_sim = max(normalized_scores)
139
+
140
+ # Coverage: how many query terms were found
141
+ coverage = matched_terms / max(query_terms, 1)
142
+
143
+ # Diversity: unique sources / total retrieved
144
+ unique_sources = len(set(sources))
145
+ diversity = unique_sources / max(len(sources), 1)
146
+
147
+ # Combined score with weights
148
+ overall = (
149
+ 0.35 * top_sim +
150
+ 0.25 * avg_sim +
151
+ 0.25 * coverage +
152
+ 0.15 * diversity
153
+ ) * self.calibration_factor
154
+
155
+ return RAGConfidence(
156
+ similarity_scores=normalized_scores,
157
+ average_similarity=avg_sim,
158
+ top_similarity=top_sim,
159
+ coverage_ratio=coverage,
160
+ source_diversity=diversity,
161
+ overall_score=min(1.0, overall)
162
+ )
163
+
164
+ def score_graph_confidence(
165
+ self,
166
+ paths_found: int,
167
+ avg_path_length: float,
168
+ node_pageranks: List[float],
169
+ edge_weights: List[float],
170
+ max_hops_used: int
171
+ ) -> GraphConfidence:
172
+ """
173
+ Calculate GraphRAG reasoning confidence.
174
+
175
+ Args:
176
+ paths_found: Number of relevant paths found
177
+ avg_path_length: Average path length in hops
178
+ node_pageranks: PageRank scores of traversed nodes
179
+ edge_weights: Weights of traversed edges
180
+ max_hops_used: Maximum hop depth used
181
+ """
182
+ if paths_found == 0:
183
+ return GraphConfidence(
184
+ path_relevance=0.0,
185
+ hop_count=0,
186
+ node_importance=0.0,
187
+ edge_strength=0.0,
188
+ reasoning_depth=0.0,
189
+ overall_score=0.0
190
+ )
191
+
192
+ # Path relevance (more paths = more confident, with diminishing returns)
193
+ path_relevance = min(1.0, paths_found / 10) * 0.8 + 0.2
194
+
195
+ # Shorter paths are more reliable
196
+ hop_score = max(0, 1.0 - (avg_path_length - 1) * 0.15)
197
+
198
+ # Node importance from PageRank
199
+ node_importance = np.mean(node_pageranks) if node_pageranks else 0.5
200
+
201
+ # Edge strength
202
+ edge_strength = np.mean(edge_weights) if edge_weights else 0.5
203
+
204
+ # Reasoning depth (deeper = potentially riskier but more insightful)
205
+ depth_factor = 1.0 - (max_hops_used - 1) * 0.1 # Penalty for deep traversal
206
+
207
+ # Combined score
208
+ overall = (
209
+ 0.30 * path_relevance +
210
+ 0.25 * hop_score +
211
+ 0.20 * node_importance +
212
+ 0.15 * edge_strength +
213
+ 0.10 * depth_factor
214
+ ) * self.calibration_factor
215
+
216
+ return GraphConfidence(
217
+ path_relevance=path_relevance,
218
+ hop_count=max_hops_used,
219
+ node_importance=node_importance,
220
+ edge_strength=edge_strength,
221
+ reasoning_depth=depth_factor,
222
+ overall_score=min(1.0, overall)
223
+ )
224
+
225
+ def score_hybrid_confidence(
226
+ self,
227
+ rag_conf: RAGConfidence,
228
+ graph_conf: GraphConfidence,
229
+ query_type: str
230
+ ) -> HybridConfidence:
231
+ """
232
+ Calculate hybrid mode confidence with intelligent weighting.
233
+
234
+ Args:
235
+ rag_conf: RAG confidence metrics
236
+ graph_conf: Graph confidence metrics
237
+ query_type: Type of query for weight adjustment
238
+ """
239
+ # Dynamic weighting based on query type
240
+ if query_type in ["causal", "relational", "predictive"]:
241
+ # Graph-heavy for reasoning queries
242
+ weight_rag = 0.3
243
+ weight_graph = 0.7
244
+ elif query_type in ["factual", "aggregation"]:
245
+ # RAG-heavy for factual queries
246
+ weight_rag = 0.7
247
+ weight_graph = 0.3
248
+ else:
249
+ # Balanced for comparison/trend
250
+ weight_rag = 0.5
251
+ weight_graph = 0.5
252
+
253
+ # Adjust weights based on individual confidence
254
+ total_conf = rag_conf.overall_score + graph_conf.overall_score
255
+ if total_conf > 0:
256
+ # Shift weight towards more confident source
257
+ rag_share = rag_conf.overall_score / total_conf
258
+ weight_rag = weight_rag * 0.5 + rag_share * 0.5
259
+ weight_graph = 1 - weight_rag
260
+
261
+ # Agreement: how similar are the confidence levels
262
+ agreement = 1 - abs(rag_conf.overall_score - graph_conf.overall_score)
263
+
264
+ # Combined score
265
+ overall = (
266
+ weight_rag * rag_conf.overall_score +
267
+ weight_graph * graph_conf.overall_score
268
+ ) * (0.8 + 0.2 * agreement) # Boost when sources agree
269
+
270
+ return HybridConfidence(
271
+ rag_confidence=rag_conf,
272
+ graph_confidence=graph_conf,
273
+ fusion_weight_rag=weight_rag,
274
+ fusion_weight_graph=weight_graph,
275
+ agreement_score=agreement,
276
+ overall_score=min(1.0, overall)
277
+ )
278
+
279
+ def score_sources(
280
+ self,
281
+ sources: List[Dict],
282
+ query: str
283
+ ) -> List[SourceScore]:
284
+ """Score individual sources for attribution"""
285
+ source_scores = []
286
+
287
+ for source in sources:
288
+ name = source.get("name", source.get("source", "Unknown"))
289
+ text = source.get("text", "")
290
+
291
+ # Relevance: keyword overlap with query
292
+ query_words = set(query.lower().split())
293
+ text_words = set(text.lower().split())
294
+ overlap = len(query_words & text_words) / max(len(query_words), 1)
295
+ relevance = min(1.0, overlap * 2)
296
+
297
+ # Coverage: length of useful content
298
+ coverage = min(1.0, len(text) / 500)
299
+
300
+ # Recency: assume recent unless dated
301
+ recency = 0.8 # Default
302
+
303
+ combined = 0.5 * relevance + 0.3 * coverage + 0.2 * recency
304
+
305
+ source_scores.append(SourceScore(
306
+ source_name=name,
307
+ relevance_score=relevance,
308
+ coverage_score=coverage,
309
+ recency_score=recency,
310
+ combined_score=combined
311
+ ))
312
+
313
+ return sorted(source_scores, key=lambda x: x.combined_score, reverse=True)
314
+
315
+ def get_confidence_level(self, score: float) -> ConfidenceLevel:
316
+ """Map score to confidence level"""
317
+ if score >= 0.9:
318
+ return ConfidenceLevel.VERY_HIGH
319
+ elif score >= 0.75:
320
+ return ConfidenceLevel.HIGH
321
+ elif score >= 0.5:
322
+ return ConfidenceLevel.MODERATE
323
+ elif score >= 0.25:
324
+ return ConfidenceLevel.LOW
325
+ else:
326
+ return ConfidenceLevel.VERY_LOW
327
+
328
+ def build_answer_confidence(
329
+ self,
330
+ mode: str,
331
+ rag_conf: Optional[RAGConfidence] = None,
332
+ graph_conf: Optional[GraphConfidence] = None,
333
+ hybrid_conf: Optional[HybridConfidence] = None,
334
+ sources: List[Dict] = None
335
+ ) -> AnswerConfidence:
336
+ """Build complete answer confidence assessment"""
337
+
338
+ # Determine overall score based on mode
339
+ if mode == "hybrid" and hybrid_conf:
340
+ overall = hybrid_conf.overall_score
341
+ elif mode == "graph" and graph_conf:
342
+ overall = graph_conf.overall_score
343
+ elif mode == "rag" and rag_conf:
344
+ overall = rag_conf.overall_score
345
+ else:
346
+ overall = 0.5 # Default moderate confidence
347
+
348
+ level = self.get_confidence_level(overall)
349
+ source_scores = self.score_sources(sources or [], "")
350
+
351
+ # Should we flag uncertainty?
352
+ should_flag = level in [ConfidenceLevel.LOW, ConfidenceLevel.VERY_LOW]
353
+
354
+ # Generate explanation
355
+ explanation = self._generate_confidence_explanation(mode, overall, level)
356
+
357
+ return AnswerConfidence(
358
+ mode=mode,
359
+ overall_score=overall,
360
+ confidence_level=level,
361
+ rag_metrics=rag_conf,
362
+ graph_metrics=graph_conf,
363
+ hybrid_metrics=hybrid_conf,
364
+ source_scores=source_scores,
365
+ explanation=explanation,
366
+ should_flag_uncertainty=should_flag
367
+ )
368
+
369
+ def _normalize_similarity(self, score: float) -> float:
370
+ """Normalize FAISS L2 distance to 0-1 similarity"""
371
+ # L2 distance: lower is better, typically 0-2 range
372
+ # Convert to similarity: 1 / (1 + distance)
373
+ if score < 0:
374
+ return 1.0
375
+ return 1.0 / (1.0 + score)
376
+
377
+ def _generate_confidence_explanation(
378
+ self,
379
+ mode: str,
380
+ score: float,
381
+ level: ConfidenceLevel
382
+ ) -> str:
383
+ """Generate human-readable confidence explanation"""
384
+ level_descriptions = {
385
+ ConfidenceLevel.VERY_HIGH: "Very high confidence based on strong evidence",
386
+ ConfidenceLevel.HIGH: "High confidence with good supporting data",
387
+ ConfidenceLevel.MODERATE: "Moderate confidence - some uncertainty exists",
388
+ ConfidenceLevel.LOW: "Low confidence - limited supporting evidence",
389
+ ConfidenceLevel.VERY_LOW: "Very low confidence - answer may be unreliable"
390
+ }
391
+
392
+ mode_descriptions = {
393
+ "rag": "document retrieval",
394
+ "graph": "knowledge graph reasoning",
395
+ "hybrid": "combined document and graph analysis",
396
+ "vision": "image analysis"
397
+ }
398
+
399
+ return (
400
+ f"{level_descriptions.get(level, 'Unknown confidence')} "
401
+ f"from {mode_descriptions.get(mode, mode)}. "
402
+ f"Score: {score:.2f}"
403
+ )
404
+
405
+
406
+ # Singleton instance
407
+ _scorer = None
408
+
409
+ def get_confidence_scorer() -> EnterpriseConfidenceScorer:
410
+ """Get or create singleton scorer"""
411
+ global _scorer
412
+ if _scorer is None:
413
+ _scorer = EnterpriseConfidenceScorer()
414
+ return _scorer
415
+
416
+
417
+ def score_answer(
418
+ mode: str,
419
+ similarity_scores: List[float] = None,
420
+ sources: List[str] = None,
421
+ paths_found: int = 0,
422
+ query_type: str = "factual"
423
+ ) -> AnswerConfidence:
424
+ """
425
+ Convenience function to score an answer.
426
+
427
+ Args:
428
+ mode: Processing mode used
429
+ similarity_scores: RAG similarity scores
430
+ sources: Source documents
431
+ paths_found: Graph paths found
432
+ query_type: Query classification type
433
+
434
+ Returns:
435
+ AnswerConfidence with complete assessment
436
+ """
437
+ scorer = get_confidence_scorer()
438
+
439
+ rag_conf = None
440
+ graph_conf = None
441
+ hybrid_conf = None
442
+
443
+ if mode in ["rag", "hybrid"]:
444
+ rag_conf = scorer.score_rag_confidence(
445
+ similarity_scores=similarity_scores or [],
446
+ sources=sources or [],
447
+ query_terms=5, # Default
448
+ matched_terms=3
449
+ )
450
+
451
+ if mode in ["graph", "hybrid"]:
452
+ graph_conf = scorer.score_graph_confidence(
453
+ paths_found=paths_found,
454
+ avg_path_length=2.0,
455
+ node_pageranks=[0.5],
456
+ edge_weights=[0.7],
457
+ max_hops_used=2
458
+ )
459
+
460
+ if mode == "hybrid" and rag_conf and graph_conf:
461
+ hybrid_conf = scorer.score_hybrid_confidence(rag_conf, graph_conf, query_type)
462
+
463
+ return scorer.build_answer_confidence(
464
+ mode=mode,
465
+ rag_conf=rag_conf,
466
+ graph_conf=graph_conf,
467
+ hybrid_conf=hybrid_conf,
468
+ sources=[{"source": s} for s in (sources or [])]
469
+ )