pollution_prediction / preprocessing /forecast_dataset.py
senyakk's picture
latest app
71ff0bf
Raw History Blame Contribute Delete
10.1 kB
import pandas as pd
import os
from preprocessing.dataset import DataSet
import streamlit as st
from datetime import datetime
from scipy.stats import ks_2samp
class ForecastMeteo(DataSet):
'''
This class is inteded for loading weather forecast dataset provided by public API
'''
def __init__(self, dataframe) -> None:
super().__init__()
self._dataframe = dataframe
self._filter_data()
def _filter_data(self) -> None:
'''
Clean the data.
'''
self._dataframe = self._dataframe.drop('date', axis=1)
self._rename_reorder() # reorder columns
self._fix_number_representation()
print(self._dataframe)
def _change_date_column(self):
"""
Converts the 'date' column into 'YYYYMMDD' and 'HH' columns.
"""
self._dataframe['date'] = pd.to_datetime(self._dataframe['date'], format='%Y-%m-%d %H:%M:%S%z', errors='coerce')
# create 'YYYYMMDD' and 'HH' columns
self._dataframe['YYYYMMDD'] = self._dataframe['date'].dt.strftime('%Y%m%d').astype(int)
self._dataframe['HH'] = self._dataframe['date'].dt.hour.astype(int)
self._dataframe['YYYYMMDD'] = self._dataframe['YYYYMMDD'].astype(int)
self._dataframe['HH'] = self._dataframe['HH'].astype(int)
# shift the range from 0-23 to 1-24 to match our training data
self._dataframe['HH'] = self._dataframe['HH'] + 1
self._dataframe = self._dataframe.drop(columns=['date']) # drop the original 'date' column
def _rename_reorder(self):
self._dataframe.rename(columns={
'wind_speed_10m': 'MWS',
'wind_gusts_10m': 'WG',
'sunshine_duration': 'SD',
'global_tilted_irradiance': 'GR',
'temperature_2m': 'T',
'dew_point_2m': 'TD'
}, inplace=True)
# reorder columns
self._dataframe = self._dataframe[['MWS', 'WG', 'T', 'TD', 'SD', 'GR']]
def _fix_number_representation(self):
"""
This method ensures consistent decimal precision and scales the columns appropriately.
"""
# T and TD converted from degrees C to 0.1 degrees C
self._dataframe['T'] = self._dataframe['T'] * 10
self._dataframe['TD'] = self._dataframe['TD'] * 10
# MWS and WG converted from m/s to 0.1 m/s
self._dataframe['MWS'] = self._dataframe['MWS'] * 10
self._dataframe['WG'] = self._dataframe['WG'] * 10
# SD converted from s to 0.1 h
self._dataframe['SD'] = self._dataframe['SD'] / 360
# GR converted from W/m2 to J/cm2
self._dataframe['GR'] = self._dataframe['GR'] * 0.36
self._dataframe = self._dataframe.round(2)
def preprocess_function(dataframe):
preprocess = ForecastMeteo(dataframe=dataframe)
return preprocess.get_df()
def split(train_test:int, train_val:int):
'''
Splits the data into training, validation, and test sets based on the
provided proportions.
'''
training_data = pd.read_csv("preprocessing/combined.csv")
X = training_data.drop(columns=["NO2", "O3", "MWD", "MWS10", "P"])
y = training_data[["NO2", "O3"]]
split_index = int(train_test * len(X))
X_train, X_test = X[:split_index], X[split_index:]
y_train, y_test = y[:split_index], y[split_index:]
split_index = int(train_val * len(X_train))
X_train, X_val = X_train[:split_index], X_train[split_index:]
y_train, y_val = y_train[:split_index], y_train[split_index:]
pd.DataFrame(X_train).to_csv('preprocessing/X_train.csv', index=False) # saving only training data split
def check_missing_values(data):
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
missing_features = []
for feature, feature_name in weather_features.items():
if feature in data and data[feature].isnull().any():
missing_features.append(feature_name)
if missing_features:
st.warning(f"⚠️ **Missing data** detected for: **{', '.join(missing_features)}.** "
"This might affect the accuracy of the forecast. 🛠️")
def subset_training_data_for_month(training_data):
"""
Subset the training data to include data only for the current month across multiple years.
"""
current_month = datetime.now().month
training_data['month'] = pd.to_datetime(training_data['YYYYMMDD'], format='%Y%m%d').dt.month
return training_data[training_data['month'] == current_month]
def detect_covariate_shift(new_data, reference_data):
"""
Detect covariate shift by comparing distributions of each feature in new data vs reference data (training data for the same month).
"""
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
shift_detected_features = []
for feature in weather_features.keys():
stat, p_value = ks_2samp(new_data[feature], reference_data[feature]) # KS test to compare the distributions
if p_value < 0.05:
shift_detected_features.append(weather_features[feature])
if shift_detected_features:
st.warning(f"🌩️ **Covariate shift detected** in the following features: **{', '.join(shift_detected_features)}**. "
"Weather patterns have changed significantly from the training data.")
def check_covariate_shift(current_data):
training_data = pd.read_csv("preprocessing/X_train.csv")
reference_data = subset_training_data_for_month(training_data)
detect_covariate_shift(current_data, reference_data)
def calculate_training_min_max():
"""
Calculate the overall minimum and maximum values for each feature in the training dataset.
"""
training_data = pd.read_csv("X_train.csv")
min_max = training_data.agg({
'MWS': ['min', 'max'],
'WG': ['min', 'max'],
'T': ['min', 'max'],
'TD': ['min', 'max'],
'SD': ['min', 'max'],
'GR': ['min', 'max'],
})
min_max.columns = ['_'.join([col, 'min' if idx == 0 else 'max']) for idx, col in enumerate(min_max.columns)]
return min_max
def calculate_monthly_ranges():
"""
Find bboundaries for each feature in the training dataset:
For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is calculated.
For T and TD: Both the lower (0.05 quantile) and upper (0.95 quantile) boundaries are calculated.
"""
full_data = pd.read_csv("preprocessing/X_train.csv")
full_data['month'] = pd.to_datetime(full_data['YYYYMMDD'], format='%Y%m%d').dt.month
monthly_percentiles = full_data.groupby('month').agg({
'MWS': [lambda x: x.quantile(0.95)],
'WG': [lambda x: x.quantile(0.95)],
'SD': [lambda x: x.quantile(0.95)],
'GR': [lambda x: x.quantile(0.95)],
'T': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)],
'TD': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)]
})
monthly_percentiles.columns = [
'MWS_95th', 'WG_95th', 'SD_95th', 'GR_95th',
'T_5th', 'T_95th',
'TD_5th', 'TD_95th']
return monthly_percentiles
def check_for_extreme_values(data):
"""
For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is checked.
For T and TD: Both lower (0.05 quantile) and upper (0.95 quantile) boundaries are checked.
"""
monthly_percentiles = calculate_monthly_ranges()
current_month = datetime.now().month
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
extreme_conditions = []
for feature in weather_features.keys():
upper_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_95th']
if feature in ['MWS', 'WG', 'SD', 'GR']:
if data[feature].max() > upper_bound_percentile:
extreme_conditions.append(weather_features[feature])
if feature in ['T', 'TD']:
lower_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_5th']
if data[feature].min() < lower_bound_percentile or data[feature].max() > upper_bound_percentile:
extreme_conditions.append(weather_features[feature])
if extreme_conditions:
st.warning(f"⚡ **Alert!** Extreme weather conditions detected for: **{', '.join(extreme_conditions)}.** "
"Some values are critical for this time of the year.")
def check_for_critical_values(data):
"""
Compare the current data against the training min-max ranges.
If values are outside the min-max range, raise an alert for unreliable data.
"""
min_max = calculate_training_min_max()
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
unreliable_data = []
for feature in weather_features.keys():
min_value = min_max.loc['min', feature] # min and max values for each feature
max_value = min_max.loc['max', feature]
if data[feature].min() < min_value or data[feature].max() > max_value:
unreliable_data.append(weather_features[feature])
if unreliable_data:
st.error(f"🚨 **Critical alert!** Data for **{', '.join(unreliable_data)}** is outside the min-max range. "
"This data is marked as unreliable and cannot be used for prediction.")
if __name__ == "__main__":
# print(preprocess_function())
# split(train_test=0.8, train_val=0.75)
calculate_monthly_ranges()