Spaces:
Sleeping
Sleeping
File size: 10,056 Bytes
307d0dd 71ff0bf f15d367 307d0dd 71ff0bf d81dba4 307d0dd 289f98e 307d0dd 289f98e 307d0dd e883258 289f98e e883258 289f98e e883258 289f98e e883258 289f98e e883258 289f98e e883258 307d0dd d81dba4 f15d367 349d439 71ff0bf 349d439 71ff0bf 349d439 f15d367 349d439 f15d367 71ff0bf f15d367 349d439 f15d367 349d439 71ff0bf 349d439 f15d367 349d439 f15d367 349d439 f15d367 349d439 f15d367 349d439 f15d367 349d439 f15d367 349d439 23659e5 349d439 f15d367 349d439 f15d367 349d439 f15d367 349d439 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 | import pandas as pd
import os
from preprocessing.dataset import DataSet
import streamlit as st
from datetime import datetime
from scipy.stats import ks_2samp
class ForecastMeteo(DataSet):
'''
This class is inteded for loading weather forecast dataset provided by public API
'''
def __init__(self, dataframe) -> None:
super().__init__()
self._dataframe = dataframe
self._filter_data()
def _filter_data(self) -> None:
'''
Clean the data.
'''
self._dataframe = self._dataframe.drop('date', axis=1)
self._rename_reorder() # reorder columns
self._fix_number_representation()
print(self._dataframe)
def _change_date_column(self):
"""
Converts the 'date' column into 'YYYYMMDD' and 'HH' columns.
"""
self._dataframe['date'] = pd.to_datetime(self._dataframe['date'], format='%Y-%m-%d %H:%M:%S%z', errors='coerce')
# create 'YYYYMMDD' and 'HH' columns
self._dataframe['YYYYMMDD'] = self._dataframe['date'].dt.strftime('%Y%m%d').astype(int)
self._dataframe['HH'] = self._dataframe['date'].dt.hour.astype(int)
self._dataframe['YYYYMMDD'] = self._dataframe['YYYYMMDD'].astype(int)
self._dataframe['HH'] = self._dataframe['HH'].astype(int)
# shift the range from 0-23 to 1-24 to match our training data
self._dataframe['HH'] = self._dataframe['HH'] + 1
self._dataframe = self._dataframe.drop(columns=['date']) # drop the original 'date' column
def _rename_reorder(self):
self._dataframe.rename(columns={
'wind_speed_10m': 'MWS',
'wind_gusts_10m': 'WG',
'sunshine_duration': 'SD',
'global_tilted_irradiance': 'GR',
'temperature_2m': 'T',
'dew_point_2m': 'TD'
}, inplace=True)
# reorder columns
self._dataframe = self._dataframe[['MWS', 'WG', 'T', 'TD', 'SD', 'GR']]
def _fix_number_representation(self):
"""
This method ensures consistent decimal precision and scales the columns appropriately.
"""
# T and TD converted from degrees C to 0.1 degrees C
self._dataframe['T'] = self._dataframe['T'] * 10
self._dataframe['TD'] = self._dataframe['TD'] * 10
# MWS and WG converted from m/s to 0.1 m/s
self._dataframe['MWS'] = self._dataframe['MWS'] * 10
self._dataframe['WG'] = self._dataframe['WG'] * 10
# SD converted from s to 0.1 h
self._dataframe['SD'] = self._dataframe['SD'] / 360
# GR converted from W/m2 to J/cm2
self._dataframe['GR'] = self._dataframe['GR'] * 0.36
self._dataframe = self._dataframe.round(2)
def preprocess_function(dataframe):
preprocess = ForecastMeteo(dataframe=dataframe)
return preprocess.get_df()
def split(train_test:int, train_val:int):
'''
Splits the data into training, validation, and test sets based on the
provided proportions.
'''
training_data = pd.read_csv("preprocessing/combined.csv")
X = training_data.drop(columns=["NO2", "O3", "MWD", "MWS10", "P"])
y = training_data[["NO2", "O3"]]
split_index = int(train_test * len(X))
X_train, X_test = X[:split_index], X[split_index:]
y_train, y_test = y[:split_index], y[split_index:]
split_index = int(train_val * len(X_train))
X_train, X_val = X_train[:split_index], X_train[split_index:]
y_train, y_val = y_train[:split_index], y_train[split_index:]
pd.DataFrame(X_train).to_csv('preprocessing/X_train.csv', index=False) # saving only training data split
def check_missing_values(data):
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
missing_features = []
for feature, feature_name in weather_features.items():
if feature in data and data[feature].isnull().any():
missing_features.append(feature_name)
if missing_features:
st.warning(f"⚠️ **Missing data** detected for: **{', '.join(missing_features)}.** "
"This might affect the accuracy of the forecast. 🛠️")
def subset_training_data_for_month(training_data):
"""
Subset the training data to include data only for the current month across multiple years.
"""
current_month = datetime.now().month
training_data['month'] = pd.to_datetime(training_data['YYYYMMDD'], format='%Y%m%d').dt.month
return training_data[training_data['month'] == current_month]
def detect_covariate_shift(new_data, reference_data):
"""
Detect covariate shift by comparing distributions of each feature in new data vs reference data (training data for the same month).
"""
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
shift_detected_features = []
for feature in weather_features.keys():
stat, p_value = ks_2samp(new_data[feature], reference_data[feature]) # KS test to compare the distributions
if p_value < 0.05:
shift_detected_features.append(weather_features[feature])
if shift_detected_features:
st.warning(f"🌩️ **Covariate shift detected** in the following features: **{', '.join(shift_detected_features)}**. "
"Weather patterns have changed significantly from the training data.")
def check_covariate_shift(current_data):
training_data = pd.read_csv("preprocessing/X_train.csv")
reference_data = subset_training_data_for_month(training_data)
detect_covariate_shift(current_data, reference_data)
def calculate_training_min_max():
"""
Calculate the overall minimum and maximum values for each feature in the training dataset.
"""
training_data = pd.read_csv("X_train.csv")
min_max = training_data.agg({
'MWS': ['min', 'max'],
'WG': ['min', 'max'],
'T': ['min', 'max'],
'TD': ['min', 'max'],
'SD': ['min', 'max'],
'GR': ['min', 'max'],
})
min_max.columns = ['_'.join([col, 'min' if idx == 0 else 'max']) for idx, col in enumerate(min_max.columns)]
return min_max
def calculate_monthly_ranges():
"""
Find bboundaries for each feature in the training dataset:
For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is calculated.
For T and TD: Both the lower (0.05 quantile) and upper (0.95 quantile) boundaries are calculated.
"""
full_data = pd.read_csv("preprocessing/X_train.csv")
full_data['month'] = pd.to_datetime(full_data['YYYYMMDD'], format='%Y%m%d').dt.month
monthly_percentiles = full_data.groupby('month').agg({
'MWS': [lambda x: x.quantile(0.95)],
'WG': [lambda x: x.quantile(0.95)],
'SD': [lambda x: x.quantile(0.95)],
'GR': [lambda x: x.quantile(0.95)],
'T': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)],
'TD': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)]
})
monthly_percentiles.columns = [
'MWS_95th', 'WG_95th', 'SD_95th', 'GR_95th',
'T_5th', 'T_95th',
'TD_5th', 'TD_95th']
return monthly_percentiles
def check_for_extreme_values(data):
"""
For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is checked.
For T and TD: Both lower (0.05 quantile) and upper (0.95 quantile) boundaries are checked.
"""
monthly_percentiles = calculate_monthly_ranges()
current_month = datetime.now().month
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
extreme_conditions = []
for feature in weather_features.keys():
upper_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_95th']
if feature in ['MWS', 'WG', 'SD', 'GR']:
if data[feature].max() > upper_bound_percentile:
extreme_conditions.append(weather_features[feature])
if feature in ['T', 'TD']:
lower_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_5th']
if data[feature].min() < lower_bound_percentile or data[feature].max() > upper_bound_percentile:
extreme_conditions.append(weather_features[feature])
if extreme_conditions:
st.warning(f"⚡ **Alert!** Extreme weather conditions detected for: **{', '.join(extreme_conditions)}.** "
"Some values are critical for this time of the year.")
def check_for_critical_values(data):
"""
Compare the current data against the training min-max ranges.
If values are outside the min-max range, raise an alert for unreliable data.
"""
min_max = calculate_training_min_max()
weather_features = {
'MWS': 'Mean Wind Speed',
'WG': 'Wind Gusts',
'T': 'Temperature',
'TD': 'Dew Point',
'SD': 'Sunshine Duration',
'GR': 'Global Tilted Irradiance'
}
unreliable_data = []
for feature in weather_features.keys():
min_value = min_max.loc['min', feature] # min and max values for each feature
max_value = min_max.loc['max', feature]
if data[feature].min() < min_value or data[feature].max() > max_value:
unreliable_data.append(weather_features[feature])
if unreliable_data:
st.error(f"🚨 **Critical alert!** Data for **{', '.join(unreliable_data)}** is outside the min-max range. "
"This data is marked as unreliable and cannot be used for prediction.")
if __name__ == "__main__":
# print(preprocess_function())
# split(train_test=0.8, train_val=0.75)
calculate_monthly_ranges()
|