| import pandas as pd | |
| import joblib | |
| from sklearn.preprocessing import LabelEncoder | |
| from sklearn.model_selection import train_test_split | |
| import os | |
| from src.logger import get_logger | |
| from src.custom_exception import CustomException | |
| logger = get_logger(__name__) | |
| class DataProcessing: | |
| def __init__(self,input_path , output_path): | |
| self.input_path = input_path | |
| self.output_path = output_path | |
| self.df = None | |
| os.makedirs(self.output_path,exist_ok=True) | |
| logger.info("Data Processing initialized..") | |
| def load_data(self): | |
| try: | |
| self.df = pd.read_csv(self.input_path) | |
| logger.info("Data loaded sucesfuly...") | |
| except Exception as e: | |
| logger.error(f"Error while loading data {e}") | |
| raise CustomException("Failed to load data" , e) | |
| def preprocess(self): | |
| try: | |
| categorical = [] | |
| numerical = [] | |
| for col in self.df.columns: | |
| if self.df[col].dtype == 'object': | |
| categorical.append(col) | |
| else: | |
| numerical.append(col) | |
| self.df["Date"] = pd.to_datetime(self.df["Date"]) | |
| self.df["Year"] = self.df["Date"].dt.year | |
| self.df["Month"] = self.df["Date"].dt.month | |
| self.df["Day"] = self.df["Date"].dt.day | |
| self.df.drop("Date" , axis=1 , inplace=True) | |
| for col in numerical: | |
| self.df[col].fillna(self.df[col].mean() , inplace=True) | |
| self.df.dropna(inplace=True) | |
| logger.info("Basic data processing done...") | |
| except Exception as e: | |
| logger.error(f"Error while preprocess data {e}") | |
| raise CustomException("Failed to preproces data" , e) | |
| def label_encode(self): | |
| try: | |
| categorical = [ | |
| 'Location', | |
| 'WindGustDir', | |
| 'WindDir9am', | |
| 'WindDir3pm', | |
| 'RainToday', | |
| 'RainTomorrow'] | |
| for col in categorical: | |
| label_encoder = LabelEncoder() | |
| self.df[col] = label_encoder.fit_transform(self.df[col]) | |
| label_mapping = dict(zip(label_encoder.classes_ , range(len(label_encoder.classes_)))) | |
| logger.info(f"Label mapping for {col}") | |
| logger.info(label_mapping) | |
| logger.info("Label Encoding DOne....") | |
| except Exception as e: | |
| logger.error(f"Error while label encode data {e}") | |
| raise CustomException("Failed to label encode data" , e) | |
| def split_data(self): | |
| try: | |
| X = self.df.drop('RainTomorrow' , axis=1) | |
| Y = self.df["RainTomorrow"] | |
| logger.info(X.columns) | |
| X_train , X_test , y_train , y_test = train_test_split(X,Y , test_size=0.2 , random_state=42) | |
| joblib.dump(X_train , os.path.join(self.output_path , "X_train.pkl")) | |
| joblib.dump(X_test , os.path.join(self.output_path , "X_test.pkl")) | |
| joblib.dump(y_train , os.path.join(self.output_path , "y_train.pkl")) | |
| joblib.dump(y_test , os.path.join(self.output_path , "y_test.pkl")) | |
| logger.info("Splitted and saved sucesfully....") | |
| except Exception as e: | |
| logger.error(f"Error while splittting data {e}") | |
| raise CustomException("Failed to split data" , e) | |
| def run(self): | |
| self.load_data() | |
| self.preprocess() | |
| self.label_encode() | |
| self.split_data() | |
| logger.info("Data Procesing Completed....") | |
| if __name__=="__main__": | |
| processor = DataProcessing("artifacts/raw/data.csv" , "artifacts/processed") | |
| processor.run() | |