import streamlit as st import pandas as pd import tensorflow as tf from huggingface_hub import hf_hub_download import numpy as np import openmeteo_requests import requests_cache from retry_requests import retry import pickle as pk import matplotlib.pyplot as plt from datetime import datetime, timedelta import matplotlib.dates as mdates import requests import logging from preprocessing.pr_featureEng import FeatureEngineeringPredict from preprocessing.forecast_dataset import preprocess_function, check_missing_values, check_for_extreme_values, check_covariate_shift from preprocessing.forecast_api import download_data # Model is cached to prevent reloading @st.cache_resource() def load_model(model_name): model_dir = hf_hub_download(repo_id="senyakk/pollution_MLP", filename=model_name) return tf.keras.models.load_model(model_dir) @st.cache_resource() def load_pca_scalers(): fitted_pca = pk.load(open("preprocessing/pca.pkl",'rb')) fitted_scaler = pk.load(open("preprocessing/scaler_features.pkl",'rb')) fitted_scalerO3_targets = pk.load(open("preprocessing/scalerO3_targets.pkl",'rb')) fitted_scalerNO2_targets = pk.load(open("preprocessing/scalerNO2_targets.pkl",'rb')) return fitted_pca, fitted_scaler, fitted_scalerO3_targets, fitted_scalerNO2_targets modelNO2 = load_model("mlpNO2.keras") modelO3 = load_model("mlpO3.keras") fitted_pca, fitted_scaler, fitted_scalerO3_targets, fitted_scalerNO2_targets = load_pca_scalers() def input_processing(data): fe_pip = FeatureEngineeringPredict(data, fitted_pca, fitted_scaler) selected_features = ['MWS', 'WG', 'T', 'TD', 'SD', 'GR'] n_steps = 3 n_steps_out = 1 fe_pip.extract_features(selected_features=selected_features, target=None, lags=None, rolling_windows=[4,12,24], drop_ogs=0) X = fe_pip.process(n_steps, n_steps_out) n_steps_predict = 72 X_input = X[-n_steps_predict:] return X_input # Function to automatically refresh weather data and predictions every 6 hours @st.cache_data(ttl=6*3600) # Cache data for 6 hours def auto_update_data_and_prediction(): # Download new weather data raw_data = download_data() if raw_data.empty: st.error(" ⚠️ We are currently experiencing issues retrieving the latest weather data. Please try again later 🚀") st.image("https://media.giphy.com/media/JIX9t2j0ZTN9S/giphy.gif", caption="Hang in there, we're working on it! 🐱") return None, None, None, None raw_data = raw_data.dropna() data = preprocess_function(raw_data) input = input_processing(data) # Make prediction predictionO3 = modelO3.predict(input) predictionO3 = fitted_scalerO3_targets.inverse_transform(predictionO3) predictionO3_flat = predictionO3.flatten() # Flatten to 1D array rounded_predictionsO3 = [round(pred, 2) for pred in predictionO3_flat] predictionNO2 = modelNO2.predict(input) predictionNO2 = fitted_scalerNO2_targets.inverse_transform(predictionNO2) predictionNO2_flat = predictionNO2.flatten() # Flatten to 1D array rounded_predictionsNO2 = [round(pred, 2) for pred in predictionNO2_flat] # Create DataFrame for predictions hours = np.arange(1, len(predictionO3_flat) + 1) prediction_df = pd.DataFrame({ "O₃ Prediction": rounded_predictionsO3, "NO₂ Prediction": rounded_predictionsNO2 }, index=hours ) # Adjust raw_data to match the length of prediction_df raw_data = raw_data.iloc[-len(prediction_df):] # Slice to match length prediction_df["Date"] = raw_data["date"].values # Add "date" column to prediction_df # Save data and predictions prediction_df.to_csv(f"data/prediction_{datetime.now().strftime('%Y-%m-%d_%H-%M')}.csv", index=False) return raw_data, data, prediction_df, datetime.now().strftime('%Y-%m-%d %H:%M:%S') # Streamlit App Interface st.title("O₃ and NO₂ Forecast Dashboard") # Adding navigation using session state if 'page' not in st.session_state: st.session_state.page = 'dashboard' def show_admin_login(): st.session_state.page = 'admin_login' def show_dashboard(): st.session_state.page = 'dashboard' def show_admin_page(): st.session_state.page = 'admin_page' # Call auto-update function to refresh data and predictions every 6 hours raw_data, hourly_dataframe, prediction_df, last_updated = auto_update_data_and_prediction() if st.session_state.page == 'dashboard': # Create a link in the upper-right corner for logging in as Admin st.sidebar.write("Log in as Admin") st.sidebar.button("Admin Login", on_click=show_admin_login) st.markdown("### Welcome to the User Dashboard!") st.markdown("This application allows to forecast O₃ and NO₂ for the next three days in the area of the city of Utrecht, the Netherlands. Based on the KNMI data obtained from the [public API](https://open-meteo.com/en/docs/knmi-api#latitude=52.11&longitude=5.1806&hourly=temperature_2m,dew_point_2m,wind_speed_10m,wind_direction_10m,wind_gusts_10m,sunshine_duration,global_tilted_irradiance&daily=weather_code).") check_missing_values(hourly_dataframe) check_for_extreme_values(hourly_dataframe[48:]) # skip first two days of history forecast check_covariate_shift(hourly_dataframe[48:]) # Check if any of the key variables are None or invalid if any(var is None for var in [raw_data, hourly_dataframe, prediction_df, last_updated]): st.error("⚠️ Data is missing or incomplete. Unable to proceed with the forecast or visualization.") st.stop() # This stops the execution of the code further down the page else: # Save the last update timestamp in session state after calling the function if 'last_updated' not in st.session_state: st.session_state['last_updated'] = last_updated # Display last update timestamp st.write(f"Last updated: {st.session_state['last_updated']}") ozone_guideline = 100 no2_guideline = 25 st.markdown(f""" ### WHO Guidelines for Air Quality - **O₃ (Ozone):** {ozone_guideline} µg/m³ (8-hour average) - **NO₂ (Nitrogen Dioxide):** {no2_guideline} µg/m³ (24-hour average) These thresholds represent the maximum recommended concentrations to ensure public health. """) #current_values = fetch_measurements() #current_values st.markdown("### Forecast") #raw_data # st.markdown("Below is the preprocessed data:") # data_placeholder = st.empty() # prediction_graph_placeholder = st.empty() # data_placeholder.write(hourly_dataframe) # Plot predictions as a graph fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(12, 15)) # Calculate the rolling averages for O₃ (8-hour) and NO₂ (24-hour) prediction_df['O₃ 8-hour Average'] = prediction_df['O₃ Prediction'].rolling(window=8).mean() prediction_df['NO₂ 24-hour Average'] = prediction_df['NO₂ Prediction'].rolling(window=24).mean() # Plot for O₃ with 8-hour rolling average ax1.plot(prediction_df["Date"], prediction_df['O₃ Prediction'], marker='o', linestyle='-', color='b', label="O₃ Prediction") ax1.plot(prediction_df["Date"], prediction_df['O₃ 8-hour Average'], marker='o', linestyle='-', color='orange', label="8-hour Average") ax1.axhline(y=ozone_guideline, color='r', linestyle='--', label="Guideline") # WHO guideline ax1.fill_between(prediction_df["Date"], ozone_guideline, prediction_df['O₃ 8-hour Average'], where=(prediction_df['O₃ 8-hour Average'] > ozone_guideline), color='red', alpha=0.3) ax1.set_xlabel('Hours', fontsize=14) ax1.set_ylabel('O₃ Prediction (µg/m³)', fontsize=14) ax1.set_title('O₃ Prediction and 8-hour Average', fontsize=16) ax1.xaxis.set_major_formatter(mdates.DateFormatter('%d-%m %H')) # Change the format to DD-MM HH ax1.legend(fontsize=12) # Plot for NO₂ with 24-hour rolling average ax2.plot(prediction_df["Date"], prediction_df['NO₂ Prediction'], marker='o', linestyle='-', color='g', label="NO₂ Prediction") ax2.plot(prediction_df["Date"], prediction_df['NO₂ 24-hour Average'], marker='o', linestyle='-', color='purple', label="24-hour Average") ax2.axhline(y=no2_guideline, color='r', linestyle='--', label="Guideline") # WHO guideline ax2.fill_between(prediction_df["Date"], no2_guideline, prediction_df['NO₂ 24-hour Average'], where=(prediction_df['NO₂ 24-hour Average'] > no2_guideline), color='red', alpha=0.3) ax2.set_xlabel('Hours', fontsize=14) ax2.set_ylabel('NO₂ Prediction (µg/m³)', fontsize=14) ax2.set_title('NO₂ Prediction and 24-hour Average', fontsize=16) ax2.xaxis.set_major_formatter(mdates.DateFormatter('%d-%m %H')) # Change the format to DD-MM HH ax2.legend(fontsize=12) # Show the combined plot st.pyplot(fig) # Option to display the table if st.checkbox("Show prediction table", value=False ): st.write("Predicted values:") st.table(prediction_df) elif st.session_state.page == 'admin_login': st.sidebar.button("Back to Dashboard", on_click=show_dashboard) st.markdown("### Admin Login") st.markdown('For demonstration purposes, every combination of username and password is valid.') username = st.text_input("Username") password = st.text_input("Password", type="password") # dummy login process if username == "admin" and password == "admin": st.success("Logged in as Admin") else: if username and password: st.error("Invalid credentials, please try again.") if username and password: st.button("Login", on_click=show_admin_page) else: st.button("Login", disabled=True) elif st.session_state.page == 'admin_page': st.sidebar.success("You are logged in as Admin") st.markdown("### Admin Page") st.markdown("Welcome to the Admin Page. Here, you can manage predictions and settings.") performance_df = pd.read_csv('results/performance.csv') month_names = ["April", "July", "October", "December"] month_map = { "April" : 4, "July" : 7, "October" : 10, "December" : 12 } month_choice = st.sidebar.selectbox("Select a month to view predictions", month_names) predictions_df = pd.read_csv(f'results/predictions{month_map[month_choice]}.csv') model_choice = ['MLP'] model_data = performance_df[performance_df['model_name'] == model_choice[0]] model_predictions = predictions_df[predictions_df['model_name'] == model_choice[0]] history_dfs_O3 = {} history_dfs_NO2 = {} for model in model_choice: history_dfs_O3[model] = pd.read_csv(f'results/history_{model}_O3.csv') history_dfs_NO2[model] = pd.read_csv(f'results/history_{model}_NO2.csv') # Display Model Performance st.header("Model Monitoring Dashboard") st.subheader(f"Current prediction model: {model_choice[0]}") col1, col2, col3 = st.columns(3) for i, model in enumerate(model_choice): with [col1, col2, col3][i]: st.subheader(f"Performance") col_o3_1, col_o3_2 = st.columns(2) with col_o3_1: st.metric("MSE (O3)", f"{model_data.loc[model_data['model_name'] == model, 'mse_loss_O3'].values[0]:.4f}") with col_o3_2: st.metric("MAE (O3)", f"{model_data.loc[model_data['model_name'] == model, 'mae_loss_O3'].values[0]:.4f}") # Second row for NO2 metrics col_no2_1, col_no2_2 = st.columns(2) with col_no2_1: st.metric("MSE (NO2)", f"{model_data.loc[model_data['model_name'] == model, 'mse_loss_NO2'].values[0]:.4f}") with col_no2_2: st.metric("MAE (NO2)", f"{model_data.loc[model_data['model_name'] == model, 'mae_loss_NO2'].values[0]:.4f}") # Plot for O3 predictions st.subheader("O3 Predictions") fig, ax = plt.subplots() time_values = model_predictions['time'].unique() actual_O3 = model_predictions.groupby('time')['actual_O3'].first().reindex(time_values) # Select number of hours num_hours = st.sidebar.slider("Select Number of Hours to Display", min_value=1, max_value=len(time_values), value=240) time_values = time_values[:num_hours] actual_O3 = actual_O3.reindex(time_values) # Plot actual O3 values ax.plot(time_values, actual_O3, label='Actual O3', color='yellowgreen', zorder=2) for model in model_choice: mdl_pred_filt = model_predictions[(model_predictions['model_name'] == model) & (model_predictions['time'].isin(time_values))] ax.plot(mdl_pred_filt['time'], mdl_pred_filt['predicted_O3'], linestyle='--', label=f'{model}', color='navy') plt.title("Actual and Predicted Values for O3") ax.set_xlabel('Time (hours)') ax.set_ylabel('O3 Levels') ax.legend() st.pyplot(fig) plt.clf() # Plot for NO2 st.subheader("NO2 Predictions") fig, ax = plt.subplots() time_values = model_predictions['time'].unique() actual_NO2 = model_predictions.groupby('time')['actual_NO2'].first().reindex(time_values) time_values = time_values[:num_hours] actual_NO2 = actual_NO2.reindex(time_values) ax.plot(time_values, actual_NO2, label='Actual', color='yellowgreen') for model in model_choice: mdl_pred_filt = model_predictions[(model_predictions['model_name'] == model) & (model_predictions['time'].isin(time_values))] ax.plot(mdl_pred_filt['time'], mdl_pred_filt['predicted_NO2'], linestyle='--', label=f'{model}', color='navy') plt.title("Actual and Predicted Values for NO2") ax.set_xlabel('Time (hours)') ax.set_ylabel('NO2 Levels') ax.legend() st.pyplot(fig) plt.clf() # Plot: Performance Over Time for O3 st.subheader("Model Performance Over Time (O3)") fig, ax = plt.subplots() for model in model_choice: ax.plot(history_dfs_O3[model]["loss"], label=f"{model} Training Loss", color='navy') ax.plot(history_dfs_O3[model]["val_loss"], linestyle='--', label=f"{model} Validation Loss", color='navy') ax.set_title("Training and Validation Loss for O3") ax.set_xlabel("Epochs") ax.set_ylabel("Loss") ax.legend() st.pyplot(fig) plt.clf() # Performance Over Time for NO2 st.subheader("Model Performance Over Time (NO2)") fig, ax = plt.subplots() for model in model_choice: ax.plot(history_dfs_NO2[model]["loss"], label=f"{model} Training Loss", color='navy') ax.plot(history_dfs_NO2[model]["val_loss"], linestyle='--', label=f"{model} Validation Loss", color='navy') ax.set_title("Training and Validation Loss for NO2") ax.set_xlabel("Epochs") ax.set_ylabel("Loss") ax.legend() st.pyplot(fig) plt.clf() st.sidebar.button("Back to Dashboard", on_click=show_dashboard) # Insert a link on the admin page st.markdown("[Go to data management page](https://www.youtube.com/watch?v=8eJZQbT0fgc)", unsafe_allow_html=True)