import pandas as pd import os from preprocessing.dataset import DataSet import streamlit as st from datetime import datetime from scipy.stats import ks_2samp class ForecastMeteo(DataSet): ''' This class is inteded for loading weather forecast dataset provided by public API ''' def __init__(self, dataframe) -> None: super().__init__() self._dataframe = dataframe self._filter_data() def _filter_data(self) -> None: ''' Clean the data. ''' self._dataframe = self._dataframe.drop('date', axis=1) self._rename_reorder() # reorder columns self._fix_number_representation() print(self._dataframe) def _change_date_column(self): """ Converts the 'date' column into 'YYYYMMDD' and 'HH' columns. """ self._dataframe['date'] = pd.to_datetime(self._dataframe['date'], format='%Y-%m-%d %H:%M:%S%z', errors='coerce') # create 'YYYYMMDD' and 'HH' columns self._dataframe['YYYYMMDD'] = self._dataframe['date'].dt.strftime('%Y%m%d').astype(int) self._dataframe['HH'] = self._dataframe['date'].dt.hour.astype(int) self._dataframe['YYYYMMDD'] = self._dataframe['YYYYMMDD'].astype(int) self._dataframe['HH'] = self._dataframe['HH'].astype(int) # shift the range from 0-23 to 1-24 to match our training data self._dataframe['HH'] = self._dataframe['HH'] + 1 self._dataframe = self._dataframe.drop(columns=['date']) # drop the original 'date' column def _rename_reorder(self): self._dataframe.rename(columns={ 'wind_speed_10m': 'MWS', 'wind_gusts_10m': 'WG', 'sunshine_duration': 'SD', 'global_tilted_irradiance': 'GR', 'temperature_2m': 'T', 'dew_point_2m': 'TD' }, inplace=True) # reorder columns self._dataframe = self._dataframe[['MWS', 'WG', 'T', 'TD', 'SD', 'GR']] def _fix_number_representation(self): """ This method ensures consistent decimal precision and scales the columns appropriately. """ # T and TD converted from degrees C to 0.1 degrees C self._dataframe['T'] = self._dataframe['T'] * 10 self._dataframe['TD'] = self._dataframe['TD'] * 10 # MWS and WG converted from m/s to 0.1 m/s self._dataframe['MWS'] = self._dataframe['MWS'] * 10 self._dataframe['WG'] = self._dataframe['WG'] * 10 # SD converted from s to 0.1 h self._dataframe['SD'] = self._dataframe['SD'] / 360 # GR converted from W/m2 to J/cm2 self._dataframe['GR'] = self._dataframe['GR'] * 0.36 self._dataframe = self._dataframe.round(2) def preprocess_function(dataframe): preprocess = ForecastMeteo(dataframe=dataframe) return preprocess.get_df() def split(train_test:int, train_val:int): ''' Splits the data into training, validation, and test sets based on the provided proportions. ''' training_data = pd.read_csv("preprocessing/combined.csv") X = training_data.drop(columns=["NO2", "O3", "MWD", "MWS10", "P"]) y = training_data[["NO2", "O3"]] split_index = int(train_test * len(X)) X_train, X_test = X[:split_index], X[split_index:] y_train, y_test = y[:split_index], y[split_index:] split_index = int(train_val * len(X_train)) X_train, X_val = X_train[:split_index], X_train[split_index:] y_train, y_val = y_train[:split_index], y_train[split_index:] pd.DataFrame(X_train).to_csv('preprocessing/X_train.csv', index=False) # saving only training data split def check_missing_values(data): weather_features = { 'MWS': 'Mean Wind Speed', 'WG': 'Wind Gusts', 'T': 'Temperature', 'TD': 'Dew Point', 'SD': 'Sunshine Duration', 'GR': 'Global Tilted Irradiance' } missing_features = [] for feature, feature_name in weather_features.items(): if feature in data and data[feature].isnull().any(): missing_features.append(feature_name) if missing_features: st.warning(f"âš ī¸ **Missing data** detected for: **{', '.join(missing_features)}.** " "This might affect the accuracy of the forecast. đŸ› ī¸") def subset_training_data_for_month(training_data): """ Subset the training data to include data only for the current month across multiple years. """ current_month = datetime.now().month training_data['month'] = pd.to_datetime(training_data['YYYYMMDD'], format='%Y%m%d').dt.month return training_data[training_data['month'] == current_month] def detect_covariate_shift(new_data, reference_data): """ Detect covariate shift by comparing distributions of each feature in new data vs reference data (training data for the same month). """ weather_features = { 'MWS': 'Mean Wind Speed', 'WG': 'Wind Gusts', 'T': 'Temperature', 'TD': 'Dew Point', 'SD': 'Sunshine Duration', 'GR': 'Global Tilted Irradiance' } shift_detected_features = [] for feature in weather_features.keys(): stat, p_value = ks_2samp(new_data[feature], reference_data[feature]) # KS test to compare the distributions if p_value < 0.05: shift_detected_features.append(weather_features[feature]) if shift_detected_features: st.warning(f"đŸŒŠī¸ **Covariate shift detected** in the following features: **{', '.join(shift_detected_features)}**. " "Weather patterns have changed significantly from the training data.") def check_covariate_shift(current_data): training_data = pd.read_csv("preprocessing/X_train.csv") reference_data = subset_training_data_for_month(training_data) detect_covariate_shift(current_data, reference_data) def calculate_training_min_max(): """ Calculate the overall minimum and maximum values for each feature in the training dataset. """ training_data = pd.read_csv("X_train.csv") min_max = training_data.agg({ 'MWS': ['min', 'max'], 'WG': ['min', 'max'], 'T': ['min', 'max'], 'TD': ['min', 'max'], 'SD': ['min', 'max'], 'GR': ['min', 'max'], }) min_max.columns = ['_'.join([col, 'min' if idx == 0 else 'max']) for idx, col in enumerate(min_max.columns)] return min_max def calculate_monthly_ranges(): """ Find bboundaries for each feature in the training dataset: For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is calculated. For T and TD: Both the lower (0.05 quantile) and upper (0.95 quantile) boundaries are calculated. """ full_data = pd.read_csv("preprocessing/X_train.csv") full_data['month'] = pd.to_datetime(full_data['YYYYMMDD'], format='%Y%m%d').dt.month monthly_percentiles = full_data.groupby('month').agg({ 'MWS': [lambda x: x.quantile(0.95)], 'WG': [lambda x: x.quantile(0.95)], 'SD': [lambda x: x.quantile(0.95)], 'GR': [lambda x: x.quantile(0.95)], 'T': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)], 'TD': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)] }) monthly_percentiles.columns = [ 'MWS_95th', 'WG_95th', 'SD_95th', 'GR_95th', 'T_5th', 'T_95th', 'TD_5th', 'TD_95th'] return monthly_percentiles def check_for_extreme_values(data): """ For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is checked. For T and TD: Both lower (0.05 quantile) and upper (0.95 quantile) boundaries are checked. """ monthly_percentiles = calculate_monthly_ranges() current_month = datetime.now().month weather_features = { 'MWS': 'Mean Wind Speed', 'WG': 'Wind Gusts', 'T': 'Temperature', 'TD': 'Dew Point', 'SD': 'Sunshine Duration', 'GR': 'Global Tilted Irradiance' } extreme_conditions = [] for feature in weather_features.keys(): upper_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_95th'] if feature in ['MWS', 'WG', 'SD', 'GR']: if data[feature].max() > upper_bound_percentile: extreme_conditions.append(weather_features[feature]) if feature in ['T', 'TD']: lower_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_5th'] if data[feature].min() < lower_bound_percentile or data[feature].max() > upper_bound_percentile: extreme_conditions.append(weather_features[feature]) if extreme_conditions: st.warning(f"⚡ **Alert!** Extreme weather conditions detected for: **{', '.join(extreme_conditions)}.** " "Some values are critical for this time of the year.") def check_for_critical_values(data): """ Compare the current data against the training min-max ranges. If values are outside the min-max range, raise an alert for unreliable data. """ min_max = calculate_training_min_max() weather_features = { 'MWS': 'Mean Wind Speed', 'WG': 'Wind Gusts', 'T': 'Temperature', 'TD': 'Dew Point', 'SD': 'Sunshine Duration', 'GR': 'Global Tilted Irradiance' } unreliable_data = [] for feature in weather_features.keys(): min_value = min_max.loc['min', feature] # min and max values for each feature max_value = min_max.loc['max', feature] if data[feature].min() < min_value or data[feature].max() > max_value: unreliable_data.append(weather_features[feature]) if unreliable_data: st.error(f"🚨 **Critical alert!** Data for **{', '.join(unreliable_data)}** is outside the min-max range. " "This data is marked as unreliable and cannot be used for prediction.") if __name__ == "__main__": # print(preprocess_function()) # split(train_test=0.8, train_val=0.75) calculate_monthly_ranges()