entropy25 commited on
Commit
52e535d
·
verified ·
1 Parent(s): e891cb3

Update data_handler.py

Browse files
Files changed (1) hide show
  1. data_handler.py +68 -0
data_handler.py CHANGED
@@ -273,6 +273,74 @@ def calculate_data_quality_score(df: pd.DataFrame) -> Dict[str, Any]:
273
  'grade': 'Unknown'
274
  }
275
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
  def load_data(uploaded_file):
277
  """Main function to load data - for compatibility"""
278
  try:
 
273
  'grade': 'Unknown'
274
  }
275
 
276
+ @st.cache_data
277
+ def get_column_types(df: pd.DataFrame) -> Dict[str, List[str]]:
278
+ """Get columns categorized by their data types"""
279
+ try:
280
+ numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
281
+ categorical_cols = df.select_dtypes(include=['object']).columns.tolist()
282
+ datetime_cols = df.select_dtypes(include=['datetime64']).columns.tolist()
283
+ boolean_cols = df.select_dtypes(include=['bool']).columns.tolist()
284
+
285
+ return {
286
+ 'numeric': numeric_cols,
287
+ 'categorical': categorical_cols,
288
+ 'datetime': datetime_cols,
289
+ 'boolean': boolean_cols,
290
+ 'all': df.columns.tolist()
291
+ }
292
+ except Exception as e:
293
+ st.error(f"Error getting column types: {str(e)}")
294
+ return {
295
+ 'numeric': [],
296
+ 'categorical': [],
297
+ 'datetime': [],
298
+ 'boolean': [],
299
+ 'all': []
300
+ }
301
+
302
+ @st.cache_data
303
+ def clean_data(df: pd.DataFrame, options: Dict[str, Any] = None) -> pd.DataFrame:
304
+ """Clean data based on specified options"""
305
+ try:
306
+ cleaned_df = df.copy()
307
+
308
+ if options is None:
309
+ options = {}
310
+
311
+ # Remove duplicates if specified
312
+ if options.get('remove_duplicates', False):
313
+ initial_rows = len(cleaned_df)
314
+ cleaned_df = cleaned_df.drop_duplicates()
315
+ removed_rows = initial_rows - len(cleaned_df)
316
+ if removed_rows > 0:
317
+ st.info(f"Removed {removed_rows} duplicate rows")
318
+
319
+ # Handle missing values
320
+ if options.get('handle_missing', False):
321
+ missing_strategy = options.get('missing_strategy', 'drop')
322
+
323
+ if missing_strategy == 'drop':
324
+ cleaned_df = cleaned_df.dropna()
325
+ elif missing_strategy == 'fill_mean':
326
+ numeric_cols = cleaned_df.select_dtypes(include=[np.number]).columns
327
+ cleaned_df[numeric_cols] = cleaned_df[numeric_cols].fillna(cleaned_df[numeric_cols].mean())
328
+ elif missing_strategy == 'fill_median':
329
+ numeric_cols = cleaned_df.select_dtypes(include=[np.number]).columns
330
+ cleaned_df[numeric_cols] = cleaned_df[numeric_cols].fillna(cleaned_df[numeric_cols].median())
331
+ elif missing_strategy == 'fill_mode':
332
+ for col in cleaned_df.columns:
333
+ if cleaned_df[col].isnull().any():
334
+ mode_value = cleaned_df[col].mode()
335
+ if not mode_value.empty:
336
+ cleaned_df[col].fillna(mode_value[0], inplace=True)
337
+
338
+ return cleaned_df
339
+
340
+ except Exception as e:
341
+ st.error(f"Error cleaning data: {str(e)}")
342
+ return df
343
+
344
  def load_data(uploaded_file):
345
  """Main function to load data - for compatibility"""
346
  try: