entropy25 commited on
Commit
6337cd5
·
verified ·
1 Parent(s): 83e7d1e

Upload 4 files

Browse files
Files changed (4) hide show
  1. .env +3 -0
  2. analyzer.py +693 -0
  3. app.py +131 -0
  4. data_handler.py +297 -0
.env ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
2
+ GOOGLE_API_KEY=AIzaSyCK3ObZ0u10m-o7cMPfj5y33m7whMCrBSU
3
+ ANTHROPIC_API_KEY=claude-xxxxxxxxxxxxxxxxxxxxxxxxxxx
analyzer.py ADDED
@@ -0,0 +1,693 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import pandas as pd
3
+ import numpy as np
4
+ import plotly.express as px
5
+ import plotly.graph_objects as go
6
+ from typing import Dict, List, Any, Optional
7
+ import os
8
+ from dotenv import load_dotenv
9
+ from data_handler import *
10
+ from io import BytesIO
11
+
12
+ # Load environment variables
13
+ load_dotenv()
14
+
15
+ # Optional AI Integration
16
+ try:
17
+ import openai
18
+ OPENAI_AVAILABLE = True
19
+ except ImportError:
20
+ OPENAI_AVAILABLE = False
21
+
22
+ try:
23
+ import google.generativeai as genai
24
+ GEMINI_AVAILABLE = True
25
+ except ImportError:
26
+ GEMINI_AVAILABLE = False
27
+
28
+ class AIAssistant:
29
+ """AI-powered analysis assistant"""
30
+
31
+ def __init__(self):
32
+ self.openai_key = os.getenv('OPENAI_API_KEY')
33
+ self.gemini_key = os.getenv('GOOGLE_API_KEY')
34
+
35
+ if self.gemini_key and GEMINI_AVAILABLE:
36
+ genai.configure(api_key=self.gemini_key)
37
+ self.gemini_model = genai.GenerativeModel('gemini-1.5-flash')
38
+
39
+ def get_available_models(self) -> List[str]:
40
+ """Get list of available AI models"""
41
+ models = []
42
+ if self.openai_key and OPENAI_AVAILABLE:
43
+ models.append("OpenAI GPT")
44
+ if self.gemini_key and GEMINI_AVAILABLE:
45
+ models.append("Google Gemini")
46
+ return models
47
+
48
+ def analyze_insights(self, df: pd.DataFrame, insights: List[Dict], model: str = "Google Gemini") -> str:
49
+ """Get AI analysis of insights"""
50
+
51
+ # Prepare data summary
52
+ summary = f"""
53
+ Dataset Summary:
54
+ - Shape: {df.shape}
55
+ - Columns: {list(df.columns)}
56
+ - Data types: {df.dtypes.value_counts().to_dict()}
57
+
58
+ Key Insights Found:
59
+ """
60
+
61
+ for insight in insights:
62
+ summary += f"\n- {insight['insight']}"
63
+
64
+ prompt = f"""
65
+ As a senior data scientist, analyze this dataset and provide:
66
+
67
+ 1. Business implications of the findings
68
+ 2. Potential opportunities or risks
69
+ 3. Recommendations for decision-making
70
+ 4. Suggestions for further analysis
71
+
72
+ {summary}
73
+
74
+ Provide actionable insights in a professional format.
75
+ """
76
+
77
+ try:
78
+ if model == "Google Gemini" and hasattr(self, 'gemini_model'):
79
+ response = self.gemini_model.generate_content(prompt)
80
+ return response.text
81
+ elif model == "OpenAI GPT" and self.openai_key:
82
+ client = openai.OpenAI(api_key=self.openai_key)
83
+ response = client.chat.completions.create(
84
+ model="gpt-3.5-turbo",
85
+ messages=[{"role": "user", "content": prompt}]
86
+ )
87
+ return response.choices[0].message.content
88
+ else:
89
+ return "AI analysis not available. Please configure API keys."
90
+ except Exception as e:
91
+ return f"AI Analysis Error: {str(e)}"
92
+
93
+ class DataAnalysisWorkflow:
94
+ """Optimized data analysis workflow with caching and pagination"""
95
+
96
+ def __init__(self, df: pd.DataFrame):
97
+ self.df = df
98
+ self.stats = calculate_basic_stats(df)
99
+ self.column_types = get_column_types(df)
100
+ self.insights = []
101
+ self.page_size = 1000 # For pagination
102
+
103
+ def add_insight(self, insight: str, stage: int):
104
+ """Add insight to analysis report"""
105
+ self.insights.append({
106
+ 'stage': stage,
107
+ 'insight': insight,
108
+ 'timestamp': pd.Timestamp.now()
109
+ })
110
+
111
+ def get_paginated_data(self, page: int = 0) -> pd.DataFrame:
112
+ """Get paginated data for display"""
113
+ start_idx = page * self.page_size
114
+ end_idx = start_idx + self.page_size
115
+ return self.df.iloc[start_idx:end_idx]
116
+
117
+ def stage_1_overview(self):
118
+ """Stage 1: Data Overview with caching"""
119
+ st.subheader("📊 Data Overview")
120
+
121
+ # Data Quality Score
122
+ quality_metrics = calculate_data_quality_score(self.df)
123
+ col1, col2, col3, col4 = st.columns(4)
124
+ with col1:
125
+ st.metric("Rows", f"{self.stats['shape'][0]:,}")
126
+ with col2:
127
+ st.metric("Columns", f"{self.stats['shape'][1]:,}")
128
+ with col3:
129
+ st.metric("Quality Score", f"{quality_metrics['score']:.1f}/100")
130
+ with col4:
131
+ st.metric("Grade", quality_metrics['grade'])
132
+
133
+ if quality_metrics['issues']:
134
+ st.warning("Quality Issues Found:")
135
+ for issue in quality_metrics['issues']:
136
+ st.write(f"• {issue}")
137
+
138
+ # Memory Usage and Optimization
139
+ st.subheader("Memory Analysis")
140
+ memory_opt = calculate_memory_optimization(self.df)
141
+ col1, col2 = st.columns(2)
142
+ with col1:
143
+ st.metric("Current Memory", f"{memory_opt['current_memory_mb']:.1f} MB")
144
+ with col2:
145
+ if memory_opt['potential_savings_mb'] > 0:
146
+ st.metric("Potential Savings",
147
+ f"{memory_opt['potential_savings_mb']:.1f} MB",
148
+ f"{memory_opt['potential_savings_pct']:.1f}%")
149
+
150
+ if st.button("Show Optimization Details"):
151
+ st.dataframe(pd.DataFrame(memory_opt['suggestions']))
152
+
153
+ # Column Cardinality Analysis
154
+ st.subheader("Column Cardinality Analysis")
155
+ cardinality_df = calculate_column_cardinality(self.df)
156
+
157
+ # Filter options
158
+ col_types = cardinality_df['Type'].unique()
159
+ selected_types = st.multiselect("Filter by Column Type",
160
+ col_types,
161
+ default=col_types)
162
+
163
+ filtered_df = cardinality_df[cardinality_df['Type'].isin(selected_types)]
164
+ st.dataframe(filtered_df, use_container_width=True)
165
+
166
+ # Highlight important findings
167
+ id_cols = filtered_df[filtered_df['Type'] == 'Unique Identifier']['Column'].tolist()
168
+ if id_cols:
169
+ st.info(f"📌 Potential ID columns found: {', '.join(id_cols)}")
170
+
171
+ const_cols = filtered_df[filtered_df['Type'] == 'Constant']['Column'].tolist()
172
+ if const_cols:
173
+ st.warning(f"⚠️ Constant columns found: {', '.join(const_cols)}")
174
+
175
+ # Data types visualization
176
+ if self.stats['dtypes']:
177
+ st.subheader("Data Types Distribution")
178
+ fig = px.pie(values=list(self.stats['dtypes'].values()),
179
+ names=list(self.stats['dtypes'].keys()),
180
+ title="Data Types")
181
+ st.plotly_chart(fig, use_container_width=True)
182
+
183
+ # Sample data with pagination
184
+ st.subheader("Sample Data")
185
+ total_pages = (len(self.df) - 1) // self.page_size + 1
186
+
187
+ if total_pages > 1:
188
+ page = st.slider("Page", 0, total_pages - 1, 0)
189
+ sample_data = self.get_paginated_data(page)
190
+ st.write(f"Showing rows {page * self.page_size + 1} to {min((page + 1) * self.page_size, len(self.df))}")
191
+ else:
192
+ sample_data = self.df.head(10)
193
+
194
+ st.dataframe(sample_data, use_container_width=True)
195
+
196
+ # Missing values analysis
197
+ missing_df = calculate_missing_data(self.df)
198
+ if not missing_df.empty:
199
+ st.subheader("Missing Values Analysis")
200
+ st.dataframe(missing_df, use_container_width=True)
201
+
202
+ worst_column = missing_df.iloc[0]['Column']
203
+ worst_percentage = missing_df.iloc[0]['Missing %']
204
+ self.add_insight(f"Column '{worst_column}' has highest missing data: {worst_percentage:.1f}%", 1)
205
+ else:
206
+ st.success("✅ No missing values found!")
207
+ self.add_insight("Dataset has no missing values - excellent data quality", 1)
208
+
209
+ # Add insights about data quality and cardinality
210
+ if quality_metrics['score'] < 80:
211
+ self.add_insight(f"Data quality needs improvement (Score: {quality_metrics['score']:.1f}/100)", 1)
212
+
213
+ if memory_opt['potential_savings_pct'] > 20:
214
+ self.add_insight(f"Potential memory optimization of {memory_opt['potential_savings_pct']:.1f}% identified", 1)
215
+
216
+ if id_cols:
217
+ self.add_insight(f"Found {len(id_cols)} potential ID columns", 1)
218
+
219
+ def stage_2_exploration(self):
220
+ """Stage 2: Exploratory Data Analysis with caching"""
221
+ st.subheader("🔍 Exploratory Data Analysis")
222
+
223
+ numeric_cols = self.column_types['numeric']
224
+ categorical_cols = self.column_types['categorical']
225
+
226
+ # Numeric analysis
227
+ if numeric_cols:
228
+ st.subheader("Numeric Variables")
229
+ selected_numeric = st.selectbox("Select numeric column:", numeric_cols)
230
+
231
+ col1, col2 = st.columns(2)
232
+ with col1:
233
+ fig = px.histogram(self.df, x=selected_numeric,
234
+ title=f"Distribution of {selected_numeric}")
235
+ st.plotly_chart(fig, use_container_width=True)
236
+
237
+ with col2:
238
+ fig = px.box(self.df, y=selected_numeric,
239
+ title=f"Box Plot of {selected_numeric}")
240
+ st.plotly_chart(fig, use_container_width=True)
241
+
242
+ # Statistical summary
243
+ st.subheader("Statistical Summary")
244
+ summary_stats = self.df[numeric_cols].describe()
245
+ st.dataframe(summary_stats, use_container_width=True)
246
+
247
+ # Correlation analysis
248
+ if len(numeric_cols) > 1:
249
+ st.subheader("Correlation Analysis")
250
+ corr_matrix = calculate_correlation_matrix(self.df)
251
+ if not corr_matrix.empty:
252
+ fig = px.imshow(corr_matrix, text_auto=True, aspect="auto",
253
+ title="Correlation Matrix")
254
+ st.plotly_chart(fig, use_container_width=True)
255
+
256
+ # Find highest correlation
257
+ corr_values = []
258
+ for i in range(len(corr_matrix.columns)):
259
+ for j in range(i+1, len(corr_matrix.columns)):
260
+ corr_values.append(abs(corr_matrix.iloc[i, j]))
261
+
262
+ if corr_values:
263
+ max_corr = max(corr_values)
264
+ self.add_insight(f"Maximum correlation coefficient: {max_corr:.3f}", 2)
265
+
266
+ # Categorical analysis
267
+ if categorical_cols:
268
+ st.subheader("Categorical Variables")
269
+ selected_categorical = st.selectbox("Select categorical column:", categorical_cols)
270
+
271
+ value_counts = get_value_counts(self.df, selected_categorical)
272
+ fig = px.bar(x=value_counts.index, y=value_counts.values,
273
+ title=f"Top 10 {selected_categorical} Values")
274
+ st.plotly_chart(fig, use_container_width=True)
275
+
276
+ total_categories = self.df[selected_categorical].nunique()
277
+ self.add_insight(f"Column '{selected_categorical}' has {total_categories} unique categories", 2)
278
+
279
+ def stage_3_cleaning(self):
280
+ """Stage 3: Data Quality Assessment"""
281
+ st.subheader("🧹 Data Quality Assessment")
282
+
283
+ cleaning_actions = []
284
+ cleaning_history = []
285
+
286
+ # Missing values handling
287
+ if self.stats['missing_values'] > 0:
288
+ st.subheader("Missing Values Treatment")
289
+ missing_df = calculate_missing_data(self.df)
290
+ st.dataframe(missing_df, use_container_width=True)
291
+
292
+ col1, col2 = st.columns(2)
293
+ with col1:
294
+ selected_col = st.selectbox("Select column to handle missing values:",
295
+ missing_df['Column'].tolist())
296
+ with col2:
297
+ fill_method = st.selectbox("Choose fill method:",
298
+ ["Drop rows", "Mean", "Median", "Mode", "Custom value"])
299
+
300
+ if st.button("Apply Missing Value Treatment"):
301
+ try:
302
+ if fill_method == "Drop rows":
303
+ self.df = self.df.dropna(subset=[selected_col])
304
+ cleaning_history.append(f"Dropped rows with missing values in {selected_col}")
305
+ else:
306
+ if fill_method == "Mean":
307
+ fill_value = self.df[selected_col].mean()
308
+ elif fill_method == "Median":
309
+ fill_value = self.df[selected_col].median()
310
+ elif fill_method == "Mode":
311
+ fill_value = self.df[selected_col].mode()[0]
312
+ else: # Custom value
313
+ fill_value = st.number_input("Enter custom value:", value=0.0)
314
+
315
+ self.df[selected_col] = self.df[selected_col].fillna(fill_value)
316
+ cleaning_history.append(f"Filled missing values in {selected_col} with {fill_method}")
317
+
318
+ st.success("✅ Missing values handled successfully!")
319
+ except Exception as e:
320
+ st.error(f"Error handling missing values: {str(e)}")
321
+
322
+ # Duplicates handling
323
+ if self.stats['duplicates'] > 0:
324
+ st.subheader("Duplicate Rows")
325
+ st.warning(f"Found {self.stats['duplicates']} duplicate rows")
326
+
327
+ if st.button("Remove Duplicate Rows"):
328
+ original_len = len(self.df)
329
+ self.df = self.df.drop_duplicates()
330
+ removed = original_len - len(self.df)
331
+ cleaning_history.append(f"Removed {removed} duplicate rows")
332
+ st.success(f"✅ Removed {removed} duplicate rows")
333
+ else:
334
+ st.success("✅ No duplicate rows found")
335
+
336
+ # Mixed type detection and handling
337
+ mixed_types = detect_mixed_types(self.df)
338
+ if mixed_types:
339
+ st.subheader("Mixed Data Types")
340
+ mixed_df = pd.DataFrame(mixed_types)
341
+ st.dataframe(mixed_df, use_container_width=True)
342
+
343
+ selected_col = st.selectbox("Select column to fix data type:",
344
+ [item['column'] for item in mixed_types])
345
+
346
+ fix_method = st.selectbox("Choose fix method:",
347
+ ["Convert to numeric", "Convert to string"])
348
+
349
+ if st.button("Fix Data Type"):
350
+ try:
351
+ if fix_method == "Convert to numeric":
352
+ self.df[selected_col] = pd.to_numeric(self.df[selected_col], errors='coerce')
353
+ else:
354
+ self.df[selected_col] = self.df[selected_col].astype(str)
355
+
356
+ cleaning_history.append(f"Fixed data type for {selected_col} to {fix_method}")
357
+ st.success("✅ Data type fixed successfully!")
358
+ except Exception as e:
359
+ st.error(f"Error fixing data type: {str(e)}")
360
+
361
+ # Outlier detection and handling
362
+ numeric_cols = self.column_types['numeric']
363
+ if numeric_cols:
364
+ st.subheader("Outlier Detection")
365
+ selected_col = st.selectbox("Select column for outlier detection:", numeric_cols)
366
+
367
+ outliers = calculate_outliers(self.df, selected_col)
368
+ outlier_count = len(outliers)
369
+
370
+ if outlier_count > 0:
371
+ st.warning(f"Found {outlier_count} potential outliers in '{selected_col}'")
372
+ st.dataframe(outliers[[selected_col]].head(100), use_container_width=True)
373
+
374
+ treatment_method = st.selectbox("Choose outlier treatment method:",
375
+ ["None", "Remove", "Cap at percentiles"])
376
+
377
+ if treatment_method != "None" and st.button("Apply Outlier Treatment"):
378
+ try:
379
+ if treatment_method == "Remove":
380
+ self.df = self.df[~self.df.index.isin(outliers.index)]
381
+ cleaning_history.append(f"Removed {outlier_count} outliers from {selected_col}")
382
+ else: # Cap at percentiles
383
+ Q1 = self.df[selected_col].quantile(0.25)
384
+ Q3 = self.df[selected_col].quantile(0.75)
385
+ IQR = Q3 - Q1
386
+ lower_bound = Q1 - 1.5 * IQR
387
+ upper_bound = Q3 + 1.5 * IQR
388
+
389
+ self.df[selected_col] = self.df[selected_col].clip(lower_bound, upper_bound)
390
+ cleaning_history.append(f"Capped outliers in {selected_col} at percentiles")
391
+
392
+ st.success("✅ Outliers handled successfully!")
393
+ except Exception as e:
394
+ st.error(f"Error handling outliers: {str(e)}")
395
+ else:
396
+ st.success(f"✅ No outliers detected in '{selected_col}'")
397
+
398
+ # Cleaning History
399
+ if cleaning_history:
400
+ st.subheader("Cleaning Operations History")
401
+ for i, operation in enumerate(cleaning_history, 1):
402
+ st.write(f"{i}. {operation}")
403
+ self.add_insight(f"Performed {len(cleaning_history)} data cleaning operations", 3)
404
+
405
+ # Summary
406
+ if cleaning_actions:
407
+ st.subheader("Remaining Action Items")
408
+ for i, action in enumerate(cleaning_actions, 1):
409
+ st.write(f"{i}. {action}")
410
+ self.add_insight(f"Identified {len(cleaning_actions)} data quality issues", 3)
411
+ else:
412
+ st.success("✅ Data quality is excellent!")
413
+ self.add_insight("No major data quality issues found", 3)
414
+
415
+ def stage_4_analysis(self):
416
+ """Stage 4: Advanced Analysis"""
417
+ st.subheader("🔬 Advanced Analysis")
418
+
419
+ numeric_cols = self.column_types['numeric']
420
+ categorical_cols = self.column_types['categorical']
421
+
422
+ # Relationship analysis
423
+ if len(numeric_cols) >= 2:
424
+ st.subheader("Variable Relationships")
425
+
426
+ col1, col2 = st.columns(2)
427
+ with col1:
428
+ x_var = st.selectbox("X Variable:", numeric_cols)
429
+ with col2:
430
+ y_var = st.selectbox("Y Variable:",
431
+ [col for col in numeric_cols if col != x_var])
432
+
433
+ # Sample data for performance if dataset is large
434
+ sample_size = min(5000, len(self.df))
435
+ sample_df = self.df.sample(n=sample_size) if len(self.df) > sample_size else self.df
436
+
437
+ fig = px.scatter(sample_df, x=x_var, y=y_var,
438
+ title=f"Relationship: {x_var} vs {y_var}")
439
+ st.plotly_chart(fig, use_container_width=True)
440
+
441
+ correlation = self.df[x_var].corr(self.df[y_var])
442
+ st.metric("Correlation", f"{correlation:.3f}")
443
+
444
+ if abs(correlation) > 0.7:
445
+ strength = "Strong"
446
+ elif abs(correlation) > 0.3:
447
+ strength = "Moderate"
448
+ else:
449
+ strength = "Weak"
450
+
451
+ direction = "positive" if correlation > 0 else "negative"
452
+ st.write(f"**Result:** {strength} {direction} correlation")
453
+ self.add_insight(f"{strength} correlation ({correlation:.3f}) between {x_var} and {y_var}", 4)
454
+
455
+ # Group analysis
456
+ if categorical_cols and numeric_cols:
457
+ st.subheader("Group Analysis")
458
+
459
+ col1, col2 = st.columns(2)
460
+ with col1:
461
+ group_var = st.selectbox("Group by:", categorical_cols)
462
+ with col2:
463
+ metric_var = st.selectbox("Analyze:", numeric_cols)
464
+
465
+ group_stats = calculate_group_stats(self.df, group_var, metric_var)
466
+ st.dataframe(group_stats, use_container_width=True)
467
+
468
+ # Sample for visualization if too many groups
469
+ unique_groups = self.df[group_var].nunique()
470
+ if unique_groups <= 20:
471
+ fig = px.box(self.df, x=group_var, y=metric_var,
472
+ title=f"{metric_var} by {group_var}")
473
+ st.plotly_chart(fig, use_container_width=True)
474
+ else:
475
+ st.info(f"Too many groups ({unique_groups}) for visualization. Showing statistics only.")
476
+
477
+ best_group = group_stats['mean'].idxmax()
478
+ best_value = group_stats.loc[best_group, 'mean']
479
+ self.add_insight(f"'{best_group}' has highest average {metric_var}: {best_value:.2f}", 4)
480
+
481
+ def stage_5_summary(self):
482
+ """Stage 5: Summary and Export"""
483
+ st.subheader("📈 Analysis Summary")
484
+
485
+ # Key metrics
486
+ col1, col2, col3 = st.columns(3)
487
+ with col1:
488
+ st.metric("Total Insights", len(self.insights))
489
+ with col2:
490
+ quality = "High" if self.stats['missing_values'] == 0 else "Medium"
491
+ st.metric("Data Quality", quality)
492
+ with col3:
493
+ st.metric("Analysis Complete", "✅")
494
+
495
+ # Insights summary
496
+ st.subheader("Key Insights")
497
+ for i, insight in enumerate(self.insights, 1):
498
+ st.write(f"{i}. **Stage {insight['stage']}:** {insight['insight']}")
499
+
500
+ # Export options
501
+ st.subheader("Export Results")
502
+ export_format = st.selectbox("Choose export format:",
503
+ ["Text Report", "Markdown Report", "Python Code", "Cleaned Data"])
504
+
505
+ if export_format == "Text Report":
506
+ report = self.generate_text_report()
507
+ st.download_button(
508
+ label="Download Text Report",
509
+ data=report,
510
+ file_name="analysis_report.txt",
511
+ mime="text/plain"
512
+ )
513
+
514
+ elif export_format == "Markdown Report":
515
+ report = self.generate_markdown_report()
516
+ st.download_button(
517
+ label="Download Markdown Report",
518
+ data=report,
519
+ file_name="analysis_report.md",
520
+ mime="text/markdown"
521
+ )
522
+
523
+ elif export_format == "Python Code":
524
+ code = self.generate_python_code()
525
+ st.code(code, language="python")
526
+ st.download_button(
527
+ label="Download Python Script",
528
+ data=code,
529
+ file_name="analysis_script.py",
530
+ mime="text/plain"
531
+ )
532
+
533
+ else: # Cleaned Data
534
+ # Offer different export formats
535
+ data_format = st.selectbox("Choose data format:",
536
+ ["CSV", "Excel", "Parquet"])
537
+
538
+ if st.button("Export Data"):
539
+ try:
540
+ if data_format == "CSV":
541
+ csv = self.df.to_csv(index=False)
542
+ st.download_button(
543
+ label="Download CSV",
544
+ data=csv,
545
+ file_name="cleaned_data.csv",
546
+ mime="text/csv"
547
+ )
548
+ elif data_format == "Excel":
549
+ excel_buffer = BytesIO()
550
+ self.df.to_excel(excel_buffer, index=False)
551
+ excel_data = excel_buffer.getvalue()
552
+ st.download_button(
553
+ label="Download Excel",
554
+ data=excel_data,
555
+ file_name="cleaned_data.xlsx",
556
+ mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
557
+ )
558
+ else: # Parquet
559
+ parquet_buffer = BytesIO()
560
+ self.df.to_parquet(parquet_buffer, index=False)
561
+ parquet_data = parquet_buffer.getvalue()
562
+ st.download_button(
563
+ label="Download Parquet",
564
+ data=parquet_data,
565
+ file_name="cleaned_data.parquet",
566
+ mime="application/octet-stream"
567
+ )
568
+ except Exception as e:
569
+ st.error(f"Error exporting data: {str(e)}")
570
+
571
+ def generate_text_report(self) -> str:
572
+ """Generate text analysis report"""
573
+ report = f"""DATA ANALYSIS REPORT
574
+ ==================
575
+
576
+ Dataset Overview:
577
+ - Rows: {self.stats['shape'][0]:,}
578
+ - Columns: {self.stats['shape'][1]:,}
579
+ - Missing Values: {self.stats['missing_values']:,}
580
+ - Memory Usage: {self.stats['memory_usage']:.1f} MB
581
+
582
+ Key Insights:
583
+ """
584
+ for insight in self.insights:
585
+ report += f"\n- Stage {insight['stage']}: {insight['insight']}"
586
+
587
+ report += f"\n\nGenerated: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}"
588
+ return report
589
+
590
+ def generate_markdown_report(self) -> str:
591
+ """Generate markdown analysis report"""
592
+ report = f"""# Data Analysis Report
593
+
594
+ ## Dataset Overview
595
+ * **Rows:** {self.stats['shape'][0]:,}
596
+ * **Columns:** {self.stats['shape'][1]:,}
597
+ * **Missing Values:** {self.stats['missing_values']:,}
598
+ * **Memory Usage:** {self.stats['memory_usage']:.1f} MB
599
+
600
+ ## Data Types
601
+ ```
602
+ {pd.DataFrame(self.stats['dtypes'].items(), columns=['Type', 'Count']).to_markdown()}
603
+ ```
604
+
605
+ ## Key Insights
606
+ """
607
+ # Group insights by stage
608
+ for stage in range(1, 6):
609
+ stage_insights = [i for i in self.insights if i['stage'] == stage]
610
+ if stage_insights:
611
+ report += f"\n### Stage {stage}\n"
612
+ for insight in stage_insights:
613
+ report += f"* {insight['insight']}\n"
614
+
615
+ report += f"\n\n*Generated: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}*"
616
+ return report
617
+
618
+ def generate_python_code(self) -> str:
619
+ """Generate reproducible Python code"""
620
+ code = """import pandas as pd
621
+ import numpy as np
622
+ import plotly.express as px
623
+ from typing import Dict, List, Any
624
+
625
+ # Load and prepare data
626
+ df = pd.read_csv('your_data.csv') # Update with your data source
627
+
628
+ # Basic statistics
629
+ def calculate_basic_stats(df: pd.DataFrame) -> Dict[str, Any]:
630
+ return {
631
+ 'shape': df.shape,
632
+ 'memory_usage': float(df.memory_usage(deep=True).sum() / 1024**2),
633
+ 'missing_values': int(df.isnull().sum().sum()),
634
+ 'dtypes': df.dtypes.value_counts().to_dict(),
635
+ 'duplicates': int(df.duplicated().sum())
636
+ }
637
+
638
+ stats = calculate_basic_stats(df)
639
+ print("\\nBasic Statistics:")
640
+ print(f"- Shape: {stats['shape']}")
641
+ print(f"- Memory Usage: {stats['memory_usage']:.1f} MB")
642
+ print(f"- Missing Values: {stats['missing_values']}")
643
+ print(f"- Duplicates: {stats['duplicates']}")
644
+
645
+ """
646
+ # Add data cleaning operations if any were performed
647
+ if hasattr(self, 'cleaning_history'):
648
+ code += "\n# Data Cleaning\n"
649
+ for operation in self.cleaning_history:
650
+ if "missing values" in operation.lower():
651
+ code += "# Handle missing values\n"
652
+ code += "df = df.fillna(method='ffill') # Update with your chosen method\n"
653
+ elif "duplicate" in operation.lower():
654
+ code += "# Remove duplicates\n"
655
+ code += "df = df.drop_duplicates()\n"
656
+ elif "outlier" in operation.lower():
657
+ code += """# Handle outliers
658
+ def remove_outliers(df: pd.DataFrame, column: str) -> pd.DataFrame:
659
+ Q1 = df[column].quantile(0.25)
660
+ Q3 = df[column].quantile(0.75)
661
+ IQR = Q3 - Q1
662
+ return df[~((df[column] < (Q1 - 1.5 * IQR)) | (df[column] > (Q3 + 1.5 * IQR)))]
663
+
664
+ # Apply to numeric columns as needed
665
+ numeric_cols = df.select_dtypes(include=[np.number]).columns
666
+ for col in numeric_cols:
667
+ df = remove_outliers(df, col)
668
+ """
669
+
670
+ # Add visualization code
671
+ code += """
672
+ # Visualizations
673
+ def plot_missing_values(df: pd.DataFrame):
674
+ missing = df.isnull().sum()
675
+ if missing.sum() > 0:
676
+ missing = missing[missing > 0]
677
+ fig = px.bar(x=missing.index, y=missing.values,
678
+ title='Missing Values by Column')
679
+ fig.show()
680
+
681
+ def plot_correlations(df: pd.DataFrame):
682
+ numeric_cols = df.select_dtypes(include=[np.number]).columns
683
+ if len(numeric_cols) > 1:
684
+ corr = df[numeric_cols].corr()
685
+ fig = px.imshow(corr, title='Correlation Matrix')
686
+ fig.show()
687
+
688
+ # Generate plots
689
+ plot_missing_values(df)
690
+ plot_correlations(df)
691
+ """
692
+
693
+ return code
app.py ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import pandas as pd
3
+ from data_handler import load_data
4
+ from analyzer import DataAnalysisWorkflow, AIAssistant
5
+
6
+ def main():
7
+ st.set_page_config(
8
+ page_title="Data Analysis Platform",
9
+ page_icon="📊",
10
+ layout="wide"
11
+ )
12
+
13
+ st.title("📊 Data Analysis Platform")
14
+ st.markdown("**Optimized workflow with caching and pagination**")
15
+
16
+ # Initialize session state
17
+ if 'current_stage' not in st.session_state:
18
+ st.session_state.current_stage = 1
19
+ if 'workflow' not in st.session_state:
20
+ st.session_state.workflow = None
21
+ if 'ai_assistant' not in st.session_state:
22
+ st.session_state.ai_assistant = AIAssistant()
23
+
24
+ # File upload
25
+ uploaded_file = st.file_uploader("Upload Dataset", type=['csv', 'xlsx'])
26
+
27
+ if uploaded_file is not None:
28
+ try:
29
+ # Load data
30
+ df = load_data(uploaded_file)
31
+ st.success(f"✅ Dataset loaded! Shape: {df.shape}")
32
+
33
+ # Initialize workflow
34
+ if st.session_state.workflow is None:
35
+ st.session_state.workflow = DataAnalysisWorkflow(df)
36
+
37
+ # Progress sidebar
38
+ st.sidebar.header("Progress")
39
+ progress = st.sidebar.progress(st.session_state.current_stage / 5)
40
+
41
+ stages = ["Data Overview", "Exploration", "Quality Check", "Analysis", "Summary"]
42
+ for i, stage in enumerate(stages, 1):
43
+ if i == st.session_state.current_stage:
44
+ st.sidebar.write(f"🔄 **{i}. {stage}**")
45
+ elif i < st.session_state.current_stage:
46
+ st.sidebar.write(f"✅ {i}. {stage}")
47
+ else:
48
+ st.sidebar.write(f"⏳ {i}. {stage}")
49
+
50
+ # Navigation
51
+ col1, col2 = st.sidebar.columns(2)
52
+ with col1:
53
+ if st.button("← Previous") and st.session_state.current_stage > 1:
54
+ st.session_state.current_stage -= 1
55
+ st.rerun()
56
+ with col2:
57
+ if st.button("Next →") and st.session_state.current_stage < 5:
58
+ st.session_state.current_stage += 1
59
+ st.rerun()
60
+
61
+ # Recent insights
62
+ st.sidebar.header("💡 Recent Insights")
63
+ recent_insights = st.session_state.workflow.insights[-3:]
64
+ for insight in recent_insights:
65
+ st.sidebar.info(f"**Stage {insight['stage']}:** {insight['insight']}")
66
+
67
+ # Main content with AI assistant
68
+ main_col, ai_col = st.columns([3, 1])
69
+
70
+ with main_col:
71
+ # Execute current stage
72
+ if st.session_state.current_stage == 1:
73
+ st.session_state.workflow.stage_1_overview()
74
+ elif st.session_state.current_stage == 2:
75
+ st.session_state.workflow.stage_2_exploration()
76
+ elif st.session_state.current_stage == 3:
77
+ st.session_state.workflow.stage_3_cleaning()
78
+ elif st.session_state.current_stage == 4:
79
+ st.session_state.workflow.stage_4_analysis()
80
+ elif st.session_state.current_stage == 5:
81
+ st.session_state.workflow.stage_5_summary()
82
+
83
+ with ai_col:
84
+ st.subheader("🤖 AI Assistant")
85
+
86
+ # AI model selection
87
+ available_models = st.session_state.ai_assistant.get_available_models()
88
+
89
+ if available_models:
90
+ selected_model = st.selectbox("AI Model:", available_models)
91
+
92
+ if st.button("Get AI Insights"):
93
+ if st.session_state.workflow.insights:
94
+ with st.spinner("Analyzing with AI..."):
95
+ ai_analysis = st.session_state.ai_assistant.analyze_insights(
96
+ df, st.session_state.workflow.insights, selected_model
97
+ )
98
+ st.write("**AI Analysis:**")
99
+ st.write(ai_analysis)
100
+ else:
101
+ st.warning("Complete some analysis stages first.")
102
+ else:
103
+ st.warning("No AI models available.")
104
+ st.info("Set GOOGLE_API_KEY or OPENAI_API_KEY environment variables.")
105
+
106
+ # Quick insights
107
+ st.subheader("📊 Quick Stats")
108
+ if st.session_state.workflow.insights:
109
+ st.metric("Total Insights", len(st.session_state.workflow.insights))
110
+ st.metric("Current Stage", f"{st.session_state.current_stage}/5")
111
+
112
+ # Latest insight
113
+ if st.session_state.workflow.insights:
114
+ latest = st.session_state.workflow.insights[-1]
115
+ st.info(f"**Latest:** {latest['insight']}")
116
+
117
+ # Data quality indicator
118
+ quality_score = 100
119
+ if st.session_state.workflow.stats['missing_values'] > 0:
120
+ quality_score -= 30
121
+ if st.session_state.workflow.stats['duplicates'] > 0:
122
+ quality_score -= 20
123
+
124
+ st.metric("Data Quality", f"{quality_score}%")
125
+
126
+ except Exception as e:
127
+ st.error(f"Error: {str(e)}")
128
+ st.info("Please check your file format and try again.")
129
+
130
+ if __name__ == "__main__":
131
+ main()
data_handler.py ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import pandas as pd
3
+ import numpy as np
4
+ import warnings
5
+ from typing import Dict, List, Any, Tuple
6
+ from scipy import stats
7
+ warnings.filterwarnings('ignore')
8
+
9
+ # All cached data processing functions
10
+ @st.cache_data
11
+ def load_csv_with_encoding(file_content: bytes, filename: str) -> pd.DataFrame:
12
+ """Load CSV with automatic encoding detection - cached"""
13
+ import chardet
14
+
15
+ detected = chardet.detect(file_content)
16
+ encoding = detected['encoding']
17
+
18
+ try:
19
+ from io import BytesIO
20
+ return pd.read_csv(BytesIO(file_content), encoding=encoding)
21
+ except:
22
+ encodings = ['utf-8', 'latin-1', 'cp1252', 'iso-8859-1']
23
+ for enc in encodings:
24
+ try:
25
+ return pd.read_csv(BytesIO(file_content), encoding=enc)
26
+ except:
27
+ continue
28
+ raise Exception("Cannot read file with any encoding")
29
+
30
+ @st.cache_data
31
+ def load_excel_file(file_content: bytes) -> pd.DataFrame:
32
+ """Load Excel file - cached"""
33
+ from io import BytesIO
34
+ return pd.read_excel(BytesIO(file_content))
35
+
36
+ @st.cache_data
37
+ def calculate_basic_stats(df: pd.DataFrame) -> Dict[str, Any]:
38
+ """Calculate basic statistics - cached"""
39
+ dtype_counts = df.dtypes.value_counts()
40
+ dtype_dict = {str(k): int(v) for k, v in dtype_counts.items()}
41
+
42
+ return {
43
+ 'shape': df.shape,
44
+ 'memory_usage': float(df.memory_usage(deep=True).sum() / 1024**2),
45
+ 'missing_values': int(df.isnull().sum().sum()),
46
+ 'dtypes': dtype_dict,
47
+ 'duplicates': int(df.duplicated().sum())
48
+ }
49
+
50
+ @st.cache_data
51
+ def calculate_column_cardinality(df: pd.DataFrame) -> pd.DataFrame:
52
+ """Calculate column cardinality analysis - cached"""
53
+ cardinality_data = []
54
+
55
+ for col in df.columns:
56
+ unique_count = df[col].nunique()
57
+ unique_ratio = unique_count / len(df)
58
+
59
+ # Determine column type based on cardinality
60
+ if unique_count == 1:
61
+ col_type = "Constant"
62
+ elif unique_count == len(df):
63
+ col_type = "Unique Identifier"
64
+ elif unique_ratio < 0.05:
65
+ col_type = "Low Cardinality"
66
+ elif unique_ratio < 0.5:
67
+ col_type = "Medium Cardinality"
68
+ else:
69
+ col_type = "High Cardinality"
70
+
71
+ cardinality_data.append({
72
+ 'Column': col,
73
+ 'Unique Count': unique_count,
74
+ 'Unique Ratio': unique_ratio,
75
+ 'Type': col_type,
76
+ 'Data Type': str(df[col].dtype)
77
+ })
78
+
79
+ return pd.DataFrame(cardinality_data)
80
+
81
+ @st.cache_data
82
+ def calculate_memory_optimization(df: pd.DataFrame) -> Dict[str, Any]:
83
+ """Calculate memory optimization suggestions - cached"""
84
+ suggestions = []
85
+ current_memory = df.memory_usage(deep=True).sum() / 1024**2
86
+ potential_savings = 0
87
+
88
+ for col in df.columns:
89
+ if df[col].dtype == 'object':
90
+ unique_ratio = df[col].nunique() / len(df)
91
+ if unique_ratio < 0.5: # Less than 50% unique values
92
+ # Estimate category memory usage
93
+ category_memory = df[col].astype('category').memory_usage(deep=True)
94
+ object_memory = df[col].memory_usage(deep=True)
95
+ savings = (object_memory - category_memory) / 1024**2
96
+
97
+ if savings > 0.1: # More than 0.1MB savings
98
+ suggestions.append({
99
+ 'column': col,
100
+ 'current_type': 'object',
101
+ 'suggested_type': 'category',
102
+ 'savings_mb': savings
103
+ })
104
+ potential_savings += savings
105
+
106
+ return {
107
+ 'suggestions': suggestions,
108
+ 'current_memory_mb': current_memory,
109
+ 'potential_savings_mb': potential_savings,
110
+ 'potential_savings_pct': (potential_savings / current_memory) * 100 if current_memory > 0 else 0
111
+ }
112
+
113
+ @st.cache_data
114
+ def calculate_missing_data(df: pd.DataFrame) -> pd.DataFrame:
115
+ """Calculate missing data analysis - cached"""
116
+ missing_data = df.isnull().sum()
117
+ if missing_data.sum() > 0:
118
+ missing_df = pd.DataFrame({
119
+ 'Column': missing_data.index,
120
+ 'Missing Count': missing_data.values,
121
+ 'Missing %': (missing_data.values / len(df)) * 100
122
+ })
123
+ return missing_df[missing_df['Missing Count'] > 0].sort_values('Missing %', ascending=False)
124
+ return pd.DataFrame()
125
+
126
+ @st.cache_data
127
+ def calculate_correlation_matrix(df: pd.DataFrame) -> pd.DataFrame:
128
+ """Calculate correlation matrix - cached"""
129
+ numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
130
+ return df[numeric_cols].corr() if len(numeric_cols) > 1 else pd.DataFrame()
131
+
132
+ @st.cache_data
133
+ def get_column_types(df: pd.DataFrame) -> Dict[str, List[str]]:
134
+ """Get column types - cached"""
135
+ return {
136
+ 'numeric': df.select_dtypes(include=[np.number]).columns.tolist(),
137
+ 'categorical': df.select_dtypes(include=['object']).columns.tolist(),
138
+ 'datetime': df.select_dtypes(include=['datetime64']).columns.tolist()
139
+ }
140
+
141
+ @st.cache_data
142
+ def calculate_numeric_stats(df: pd.DataFrame, column: str) -> Dict[str, float]:
143
+ """Calculate enhanced numeric statistics - cached"""
144
+ series = df[column].dropna()
145
+ return {
146
+ 'mean': series.mean(),
147
+ 'median': series.median(),
148
+ 'std': series.std(),
149
+ 'skewness': series.skew(),
150
+ 'kurtosis': series.kurtosis(),
151
+ 'min': series.min(),
152
+ 'max': series.max(),
153
+ 'q25': series.quantile(0.25),
154
+ 'q75': series.quantile(0.75)
155
+ }
156
+
157
+ @st.cache_data
158
+ def calculate_outliers(df: pd.DataFrame, column: str) -> pd.DataFrame:
159
+ """Calculate outliers using IQR method - cached"""
160
+ Q1 = df[column].quantile(0.25)
161
+ Q3 = df[column].quantile(0.75)
162
+ IQR = Q3 - Q1
163
+ lower_bound = Q1 - 1.5 * IQR
164
+ upper_bound = Q3 + 1.5 * IQR
165
+
166
+ return df[(df[column] < lower_bound) | (df[column] > upper_bound)]
167
+
168
+ @st.cache_data
169
+ def detect_mixed_types(df: pd.DataFrame) -> List[Dict[str, Any]]:
170
+ """Detect columns with mixed data types - cached"""
171
+ mixed_type_issues = []
172
+
173
+ for col in df.select_dtypes(include=['object']).columns:
174
+ # Try to convert to numeric
175
+ numeric_conversion = pd.to_numeric(df[col], errors='coerce')
176
+ new_nulls = numeric_conversion.isnull().sum() - df[col].isnull().sum()
177
+
178
+ if new_nulls > 0:
179
+ mixed_type_issues.append({
180
+ 'column': col,
181
+ 'problematic_values': new_nulls,
182
+ 'total_values': len(df[col]),
183
+ 'percentage': (new_nulls / len(df[col])) * 100
184
+ })
185
+
186
+ return mixed_type_issues
187
+
188
+ @st.cache_data
189
+ def get_value_counts(df: pd.DataFrame, column: str, top_n: int = 10) -> pd.Series:
190
+ """Get value counts for categorical column - cached"""
191
+ return df[column].value_counts().head(top_n)
192
+
193
+ @st.cache_data
194
+ def calculate_crosstab(df: pd.DataFrame, col1: str, col2: str) -> pd.DataFrame:
195
+ """Calculate crosstab between two categorical columns - cached"""
196
+ return pd.crosstab(df[col1], df[col2])
197
+
198
+ @st.cache_data
199
+ def calculate_group_stats(df: pd.DataFrame, group_col: str, metric_col: str) -> pd.DataFrame:
200
+ """Calculate group statistics - cached"""
201
+ return df.groupby(group_col)[metric_col].agg(['mean', 'median', 'std', 'count'])
202
+
203
+ @st.cache_data
204
+ def calculate_data_quality_score(df: pd.DataFrame) -> Dict[str, Any]:
205
+ """Calculate overall data quality score - cached"""
206
+ score = 100
207
+ issues = []
208
+
209
+ # Missing values penalty
210
+ missing_pct = (df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100
211
+ if missing_pct > 0:
212
+ penalty = min(30, missing_pct * 2) # Max 30 points penalty
213
+ score -= penalty
214
+ issues.append(f"Missing values: {missing_pct:.1f}%")
215
+
216
+ # Duplicates penalty
217
+ duplicate_pct = (df.duplicated().sum() / len(df)) * 100
218
+ if duplicate_pct > 0:
219
+ penalty = min(20, duplicate_pct * 4) # Max 20 points penalty
220
+ score -= penalty
221
+ issues.append(f"Duplicate rows: {duplicate_pct:.1f}%")
222
+
223
+ # Constant columns penalty
224
+ constant_cols = [col for col in df.columns if df[col].nunique() == 1]
225
+ if constant_cols:
226
+ penalty = min(10, len(constant_cols) * 2)
227
+ score -= penalty
228
+ issues.append(f"Constant columns: {len(constant_cols)}")
229
+
230
+ # Mixed types penalty
231
+ mixed_types = detect_mixed_types(df)
232
+ if mixed_types:
233
+ penalty = min(10, len(mixed_types) * 3)
234
+ score -= penalty
235
+ issues.append(f"Mixed type columns: {len(mixed_types)}")
236
+
237
+ return {
238
+ 'score': max(0, score),
239
+ 'issues': issues,
240
+ 'grade': 'A' if score >= 90 else 'B' if score >= 80 else 'C' if score >= 70 else 'D' if score >= 60 else 'F'
241
+ }
242
+
243
+ def load_data(uploaded_file):
244
+ """Unified data loading function"""
245
+ file_content = uploaded_file.read()
246
+ uploaded_file.seek(0)
247
+
248
+ if uploaded_file.name.endswith('.csv'):
249
+ return load_csv_with_encoding(file_content, uploaded_file.name)
250
+ else:
251
+ return load_excel_file(file_content)
252
+
253
+ def apply_data_cleaning(df: pd.DataFrame, operations: List[Dict[str, Any]]) -> pd.DataFrame:
254
+ """Apply data cleaning operations"""
255
+ cleaned_df = df.copy()
256
+
257
+ for operation in operations:
258
+ if operation['type'] == 'fill_missing':
259
+ if operation['method'] == 'mean':
260
+ cleaned_df[operation['column']] = cleaned_df[operation['column']].fillna(
261
+ cleaned_df[operation['column']].mean())
262
+ elif operation['method'] == 'median':
263
+ cleaned_df[operation['column']] = cleaned_df[operation['column']].fillna(
264
+ cleaned_df[operation['column']].median())
265
+ elif operation['method'] == 'mode':
266
+ cleaned_df[operation['column']] = cleaned_df[operation['column']].fillna(
267
+ cleaned_df[operation['column']].mode().iloc[0] if not cleaned_df[operation['column']].mode().empty else 0)
268
+ elif operation['method'] == 'drop':
269
+ cleaned_df = cleaned_df.dropna(subset=[operation['column']])
270
+
271
+ elif operation['type'] == 'remove_duplicates':
272
+ cleaned_df = cleaned_df.drop_duplicates()
273
+
274
+ elif operation['type'] == 'remove_outliers':
275
+ Q1 = cleaned_df[operation['column']].quantile(0.25)
276
+ Q3 = cleaned_df[operation['column']].quantile(0.75)
277
+ IQR = Q3 - Q1
278
+ lower_bound = Q1 - 1.5 * IQR
279
+ upper_bound = Q3 + 1.5 * IQR
280
+ cleaned_df = cleaned_df[
281
+ (cleaned_df[operation['column']] >= lower_bound) &
282
+ (cleaned_df[operation['column']] <= upper_bound)
283
+ ]
284
+
285
+ elif operation['type'] == 'cap_outliers':
286
+ Q1 = cleaned_df[operation['column']].quantile(0.25)
287
+ Q3 = cleaned_df[operation['column']].quantile(0.75)
288
+ IQR = Q3 - Q1
289
+ lower_bound = Q1 - 1.5 * IQR
290
+ upper_bound = Q3 + 1.5 * IQR
291
+ cleaned_df[operation['column']] = cleaned_df[operation['column']].clip(lower_bound, upper_bound)
292
+
293
+ elif operation['type'] == 'convert_type':
294
+ if operation['target_type'] == 'category':
295
+ cleaned_df[operation['column']] = cleaned_df[operation['column']].astype('category')
296
+
297
+ return cleaned_df