| |
| from loguru import logger |
| import pandas as pd |
| import numpy as np |
| from dataclasses import dataclass |
| from typing import Dict, List, Optional, Tuple |
| import json |
| from pathlib import Path |
| import logging |
| from scipy.stats import entropy |
| import warnings |
| from datetime import datetime |
| import matplotlib.pyplot as plt |
| import seaborn as sns |
| import re |
| from textblob import TextBlob |
| import os |
| from pathlib import Path |
|
|
| class DataCleaner: |
| def __init__(self,df,output_path,filename="preprocessed_cleaned.csv"): |
| self.df=df |
| self.output_path=output_path |
| self.filename=filename |
| def saving_cleaned_preprocess(self): |
| Path(self.output_path).mkdir(parents=True, exist_ok=True) |
| |
| output_file = Path(self.output_path) / self.filename |
| logger.info(f"Files saved in directory {output_file} as : { self.filename}") |
| self.df.to_csv(output_file, index=False) |
| |
| def dropping_unncessary_columns(self): |
| self.df.drop("review_text", axis=1, inplace=True) |
| self.df.drop("review_date", axis=1, inplace=True) |
| self.df.drop("business_name", axis=1, inplace=True) |
| self.df.drop("address", axis=1, inplace=True) |
| self.df.drop("city", axis=1, inplace=True) |
| self.df.drop("state", axis=1, inplace=True) |
| self.df.drop("postal_code", axis=1, inplace=True) |
| self.df.drop("categories", axis=1, inplace=True) |
| self.df.drop("user_name", axis=1, inplace=True) |
| self.df.drop("yelping_since", axis=1, inplace=True) |
| self.df.drop("checkin_date", axis=1, inplace=True) |
| self.df.drop("review_useful", axis=1, inplace=True) |
| self.df.drop("review_funny", axis=1, inplace=True) |
| self.df.drop("review_cool", axis=1, inplace=True) |
| self.df.drop("user_useful", axis=1, inplace=True) |
| self.df.drop("user_funny", axis=1, inplace=True) |
| self.df.drop("user_cool", axis=1, inplace=True) |
| self.df.drop("is_open", axis=1, inplace=True) |
| self.df.drop("compliment_hot", axis=1, inplace=True) |
| self.df.drop("compliment_more", axis=1, inplace=True) |
| self.df.drop("compliment_profile", axis=1, inplace=True) |
| self.df.drop("compliment_cute", axis=1, inplace=True) |
| self.df.drop("compliment_list", axis=1, inplace=True) |
| self.df.drop("compliment_note", axis=1, inplace=True) |
| self.df.drop("compliment_plain", axis=1, inplace=True) |
| self.df.drop("compliment_cool", axis=1, inplace=True) |
| self.df.drop("compliment_funny", axis=1, inplace=True) |
| self.df.drop("compliment_writer", axis=1, inplace=True) |
| self.df.drop("compliment_photos", axis=1, inplace=True) |
|
|
| def run_pipeline(self): |
| logger.info("Dropping Unnecessary Columns") |
| self.dropping_unncessary_columns() |
|
|
| logger.info("Checking Again for NULL values in Columns") |
| for col in self.df.columns: |
| if self.df[col].isnull().sum()>0: |
| print(f" {col} has {self.df[col].isnull().sum()} null values") |
|
|
| |
| logger.info("Saving Cleaned and Preprocessed Data") |
| self.saving_cleaned_preprocess() |
| |
| |
| |
|
|
|
|
| |
| |
| |
| |
