Spaces:
Sleeping
Sleeping
Download preprocessing/forecast_dataset.py from MLINPrediction/pollution_prediction: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/spaces/MLINPrediction/pollution_prediction/resolve/main/preprocessing/forecast_dataset.py
- Command line
-
hf download hf://spaces/MLINPrediction/pollution_prediction/preprocessing/forecast_dataset.py
-
curl -L -o forecast_dataset.py https://huggingface.co/spaces/MLINPrediction/pollution_prediction/resolve/main/preprocessing/forecast_dataset.py
10.1 kB
| import pandas as pd | |
| import os | |
| from preprocessing.dataset import DataSet | |
| import streamlit as st | |
| from datetime import datetime | |
| from scipy.stats import ks_2samp | |
| class ForecastMeteo(DataSet): | |
| ''' | |
| This class is inteded for loading weather forecast dataset provided by public API | |
| ''' | |
| def __init__(self, dataframe) -> None: | |
| super().__init__() | |
| self._dataframe = dataframe | |
| self._filter_data() | |
| def _filter_data(self) -> None: | |
| ''' | |
| Clean the data. | |
| ''' | |
| self._dataframe = self._dataframe.drop('date', axis=1) | |
| self._rename_reorder() # reorder columns | |
| self._fix_number_representation() | |
| print(self._dataframe) | |
| def _change_date_column(self): | |
| """ | |
| Converts the 'date' column into 'YYYYMMDD' and 'HH' columns. | |
| """ | |
| self._dataframe['date'] = pd.to_datetime(self._dataframe['date'], format='%Y-%m-%d %H:%M:%S%z', errors='coerce') | |
| # create 'YYYYMMDD' and 'HH' columns | |
| self._dataframe['YYYYMMDD'] = self._dataframe['date'].dt.strftime('%Y%m%d').astype(int) | |
| self._dataframe['HH'] = self._dataframe['date'].dt.hour.astype(int) | |
| self._dataframe['YYYYMMDD'] = self._dataframe['YYYYMMDD'].astype(int) | |
| self._dataframe['HH'] = self._dataframe['HH'].astype(int) | |
| # shift the range from 0-23 to 1-24 to match our training data | |
| self._dataframe['HH'] = self._dataframe['HH'] + 1 | |
| self._dataframe = self._dataframe.drop(columns=['date']) # drop the original 'date' column | |
| def _rename_reorder(self): | |
| self._dataframe.rename(columns={ | |
| 'wind_speed_10m': 'MWS', | |
| 'wind_gusts_10m': 'WG', | |
| 'sunshine_duration': 'SD', | |
| 'global_tilted_irradiance': 'GR', | |
| 'temperature_2m': 'T', | |
| 'dew_point_2m': 'TD' | |
| }, inplace=True) | |
| # reorder columns | |
| self._dataframe = self._dataframe[['MWS', 'WG', 'T', 'TD', 'SD', 'GR']] | |
| def _fix_number_representation(self): | |
| """ | |
| This method ensures consistent decimal precision and scales the columns appropriately. | |
| """ | |
| # T and TD converted from degrees C to 0.1 degrees C | |
| self._dataframe['T'] = self._dataframe['T'] * 10 | |
| self._dataframe['TD'] = self._dataframe['TD'] * 10 | |
| # MWS and WG converted from m/s to 0.1 m/s | |
| self._dataframe['MWS'] = self._dataframe['MWS'] * 10 | |
| self._dataframe['WG'] = self._dataframe['WG'] * 10 | |
| # SD converted from s to 0.1 h | |
| self._dataframe['SD'] = self._dataframe['SD'] / 360 | |
| # GR converted from W/m2 to J/cm2 | |
| self._dataframe['GR'] = self._dataframe['GR'] * 0.36 | |
| self._dataframe = self._dataframe.round(2) | |
| def preprocess_function(dataframe): | |
| preprocess = ForecastMeteo(dataframe=dataframe) | |
| return preprocess.get_df() | |
| def split(train_test:int, train_val:int): | |
| ''' | |
| Splits the data into training, validation, and test sets based on the | |
| provided proportions. | |
| ''' | |
| training_data = pd.read_csv("preprocessing/combined.csv") | |
| X = training_data.drop(columns=["NO2", "O3", "MWD", "MWS10", "P"]) | |
| y = training_data[["NO2", "O3"]] | |
| split_index = int(train_test * len(X)) | |
| X_train, X_test = X[:split_index], X[split_index:] | |
| y_train, y_test = y[:split_index], y[split_index:] | |
| split_index = int(train_val * len(X_train)) | |
| X_train, X_val = X_train[:split_index], X_train[split_index:] | |
| y_train, y_val = y_train[:split_index], y_train[split_index:] | |
| pd.DataFrame(X_train).to_csv('preprocessing/X_train.csv', index=False) # saving only training data split | |
| def check_missing_values(data): | |
| weather_features = { | |
| 'MWS': 'Mean Wind Speed', | |
| 'WG': 'Wind Gusts', | |
| 'T': 'Temperature', | |
| 'TD': 'Dew Point', | |
| 'SD': 'Sunshine Duration', | |
| 'GR': 'Global Tilted Irradiance' | |
| } | |
| missing_features = [] | |
| for feature, feature_name in weather_features.items(): | |
| if feature in data and data[feature].isnull().any(): | |
| missing_features.append(feature_name) | |
| if missing_features: | |
| st.warning(f"⚠️ **Missing data** detected for: **{', '.join(missing_features)}.** " | |
| "This might affect the accuracy of the forecast. 🛠️") | |
| def subset_training_data_for_month(training_data): | |
| """ | |
| Subset the training data to include data only for the current month across multiple years. | |
| """ | |
| current_month = datetime.now().month | |
| training_data['month'] = pd.to_datetime(training_data['YYYYMMDD'], format='%Y%m%d').dt.month | |
| return training_data[training_data['month'] == current_month] | |
| def detect_covariate_shift(new_data, reference_data): | |
| """ | |
| Detect covariate shift by comparing distributions of each feature in new data vs reference data (training data for the same month). | |
| """ | |
| weather_features = { | |
| 'MWS': 'Mean Wind Speed', | |
| 'WG': 'Wind Gusts', | |
| 'T': 'Temperature', | |
| 'TD': 'Dew Point', | |
| 'SD': 'Sunshine Duration', | |
| 'GR': 'Global Tilted Irradiance' | |
| } | |
| shift_detected_features = [] | |
| for feature in weather_features.keys(): | |
| stat, p_value = ks_2samp(new_data[feature], reference_data[feature]) # KS test to compare the distributions | |
| if p_value < 0.05: | |
| shift_detected_features.append(weather_features[feature]) | |
| if shift_detected_features: | |
| st.warning(f"🌩️ **Covariate shift detected** in the following features: **{', '.join(shift_detected_features)}**. " | |
| "Weather patterns have changed significantly from the training data.") | |
| def check_covariate_shift(current_data): | |
| training_data = pd.read_csv("preprocessing/X_train.csv") | |
| reference_data = subset_training_data_for_month(training_data) | |
| detect_covariate_shift(current_data, reference_data) | |
| def calculate_training_min_max(): | |
| """ | |
| Calculate the overall minimum and maximum values for each feature in the training dataset. | |
| """ | |
| training_data = pd.read_csv("X_train.csv") | |
| min_max = training_data.agg({ | |
| 'MWS': ['min', 'max'], | |
| 'WG': ['min', 'max'], | |
| 'T': ['min', 'max'], | |
| 'TD': ['min', 'max'], | |
| 'SD': ['min', 'max'], | |
| 'GR': ['min', 'max'], | |
| }) | |
| min_max.columns = ['_'.join([col, 'min' if idx == 0 else 'max']) for idx, col in enumerate(min_max.columns)] | |
| return min_max | |
| def calculate_monthly_ranges(): | |
| """ | |
| Find bboundaries for each feature in the training dataset: | |
| For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is calculated. | |
| For T and TD: Both the lower (0.05 quantile) and upper (0.95 quantile) boundaries are calculated. | |
| """ | |
| full_data = pd.read_csv("preprocessing/X_train.csv") | |
| full_data['month'] = pd.to_datetime(full_data['YYYYMMDD'], format='%Y%m%d').dt.month | |
| monthly_percentiles = full_data.groupby('month').agg({ | |
| 'MWS': [lambda x: x.quantile(0.95)], | |
| 'WG': [lambda x: x.quantile(0.95)], | |
| 'SD': [lambda x: x.quantile(0.95)], | |
| 'GR': [lambda x: x.quantile(0.95)], | |
| 'T': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)], | |
| 'TD': [lambda x: x.quantile(0.05), lambda x: x.quantile(0.95)] | |
| }) | |
| monthly_percentiles.columns = [ | |
| 'MWS_95th', 'WG_95th', 'SD_95th', 'GR_95th', | |
| 'T_5th', 'T_95th', | |
| 'TD_5th', 'TD_95th'] | |
| return monthly_percentiles | |
| def check_for_extreme_values(data): | |
| """ | |
| For MWS, WG, SD, GR: Only the upper boundary (0.95 quantile) is checked. | |
| For T and TD: Both lower (0.05 quantile) and upper (0.95 quantile) boundaries are checked. | |
| """ | |
| monthly_percentiles = calculate_monthly_ranges() | |
| current_month = datetime.now().month | |
| weather_features = { | |
| 'MWS': 'Mean Wind Speed', | |
| 'WG': 'Wind Gusts', | |
| 'T': 'Temperature', | |
| 'TD': 'Dew Point', | |
| 'SD': 'Sunshine Duration', | |
| 'GR': 'Global Tilted Irradiance' | |
| } | |
| extreme_conditions = [] | |
| for feature in weather_features.keys(): | |
| upper_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_95th'] | |
| if feature in ['MWS', 'WG', 'SD', 'GR']: | |
| if data[feature].max() > upper_bound_percentile: | |
| extreme_conditions.append(weather_features[feature]) | |
| if feature in ['T', 'TD']: | |
| lower_bound_percentile = monthly_percentiles.loc[current_month, f'{feature}_5th'] | |
| if data[feature].min() < lower_bound_percentile or data[feature].max() > upper_bound_percentile: | |
| extreme_conditions.append(weather_features[feature]) | |
| if extreme_conditions: | |
| st.warning(f"⚡ **Alert!** Extreme weather conditions detected for: **{', '.join(extreme_conditions)}.** " | |
| "Some values are critical for this time of the year.") | |
| def check_for_critical_values(data): | |
| """ | |
| Compare the current data against the training min-max ranges. | |
| If values are outside the min-max range, raise an alert for unreliable data. | |
| """ | |
| min_max = calculate_training_min_max() | |
| weather_features = { | |
| 'MWS': 'Mean Wind Speed', | |
| 'WG': 'Wind Gusts', | |
| 'T': 'Temperature', | |
| 'TD': 'Dew Point', | |
| 'SD': 'Sunshine Duration', | |
| 'GR': 'Global Tilted Irradiance' | |
| } | |
| unreliable_data = [] | |
| for feature in weather_features.keys(): | |
| min_value = min_max.loc['min', feature] # min and max values for each feature | |
| max_value = min_max.loc['max', feature] | |
| if data[feature].min() < min_value or data[feature].max() > max_value: | |
| unreliable_data.append(weather_features[feature]) | |
| if unreliable_data: | |
| st.error(f"🚨 **Critical alert!** Data for **{', '.join(unreliable_data)}** is outside the min-max range. " | |
| "This data is marked as unreliable and cannot be used for prediction.") | |
| if __name__ == "__main__": | |
| # print(preprocess_function()) | |
| # split(train_test=0.8, train_val=0.75) | |
| calculate_monthly_ranges() | |