File size: 3,926 Bytes
07e7b9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | import pandas as pd
import joblib
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
import os
from src.logger import get_logger
from src.custom_exception import CustomException
logger = get_logger(__name__)
class DataProcessing:
def __init__(self,input_path , output_path):
self.input_path = input_path
self.output_path = output_path
self.df = None
os.makedirs(self.output_path,exist_ok=True)
logger.info("Data Processing initialized..")
def load_data(self):
try:
self.df = pd.read_csv(self.input_path)
logger.info("Data loaded sucesfuly...")
except Exception as e:
logger.error(f"Error while loading data {e}")
raise CustomException("Failed to load data" , e)
def preprocess(self):
try:
categorical = []
numerical = []
for col in self.df.columns:
if self.df[col].dtype == 'object':
categorical.append(col)
else:
numerical.append(col)
self.df["Date"] = pd.to_datetime(self.df["Date"])
self.df["Year"] = self.df["Date"].dt.year
self.df["Month"] = self.df["Date"].dt.month
self.df["Day"] = self.df["Date"].dt.day
self.df.drop("Date" , axis=1 , inplace=True)
for col in numerical:
self.df[col].fillna(self.df[col].mean() , inplace=True)
self.df.dropna(inplace=True)
logger.info("Basic data processing done...")
except Exception as e:
logger.error(f"Error while preprocess data {e}")
raise CustomException("Failed to preproces data" , e)
def label_encode(self):
try:
categorical = [
'Location',
'WindGustDir',
'WindDir9am',
'WindDir3pm',
'RainToday',
'RainTomorrow']
for col in categorical:
label_encoder = LabelEncoder()
self.df[col] = label_encoder.fit_transform(self.df[col])
label_mapping = dict(zip(label_encoder.classes_ , range(len(label_encoder.classes_))))
logger.info(f"Label mapping for {col}")
logger.info(label_mapping)
logger.info("Label Encoding DOne....")
except Exception as e:
logger.error(f"Error while label encode data {e}")
raise CustomException("Failed to label encode data" , e)
def split_data(self):
try:
X = self.df.drop('RainTomorrow' , axis=1)
Y = self.df["RainTomorrow"]
logger.info(X.columns)
X_train , X_test , y_train , y_test = train_test_split(X,Y , test_size=0.2 , random_state=42)
joblib.dump(X_train , os.path.join(self.output_path , "X_train.pkl"))
joblib.dump(X_test , os.path.join(self.output_path , "X_test.pkl"))
joblib.dump(y_train , os.path.join(self.output_path , "y_train.pkl"))
joblib.dump(y_test , os.path.join(self.output_path , "y_test.pkl"))
logger.info("Splitted and saved sucesfully....")
except Exception as e:
logger.error(f"Error while splittting data {e}")
raise CustomException("Failed to split data" , e)
def run(self):
self.load_data()
self.preprocess()
self.label_encode()
self.split_data()
logger.info("Data Procesing Completed....")
if __name__=="__main__":
processor = DataProcessing("artifacts/raw/data.csv" , "artifacts/processed")
processor.run()
|