senyakk's picture
latest app
71ff0bf
Raw History Blame Contribute Delete
15.5 kB
import streamlit as st
import pandas as pd
import tensorflow as tf
from huggingface_hub import hf_hub_download
import numpy as np
import openmeteo_requests
import requests_cache
from retry_requests import retry
import pickle as pk
import matplotlib.pyplot as plt
from datetime import datetime, timedelta
import matplotlib.dates as mdates
import requests
import logging
from preprocessing.pr_featureEng import FeatureEngineeringPredict
from preprocessing.forecast_dataset import preprocess_function, check_missing_values, check_for_extreme_values, check_covariate_shift
from preprocessing.forecast_api import download_data
# Model is cached to prevent reloading
@st.cache_resource()
def load_model(model_name):
model_dir = hf_hub_download(repo_id="senyakk/pollution_MLP", filename=model_name)
return tf.keras.models.load_model(model_dir)
@st.cache_resource()
def load_pca_scalers():
fitted_pca = pk.load(open("preprocessing/pca.pkl",'rb'))
fitted_scaler = pk.load(open("preprocessing/scaler_features.pkl",'rb'))
fitted_scalerO3_targets = pk.load(open("preprocessing/scalerO3_targets.pkl",'rb'))
fitted_scalerNO2_targets = pk.load(open("preprocessing/scalerNO2_targets.pkl",'rb'))
return fitted_pca, fitted_scaler, fitted_scalerO3_targets, fitted_scalerNO2_targets
modelNO2 = load_model("mlpNO2.keras")
modelO3 = load_model("mlpO3.keras")
fitted_pca, fitted_scaler, fitted_scalerO3_targets, fitted_scalerNO2_targets = load_pca_scalers()
def input_processing(data):
fe_pip = FeatureEngineeringPredict(data, fitted_pca, fitted_scaler)
selected_features = ['MWS', 'WG', 'T', 'TD', 'SD', 'GR']
n_steps = 3
n_steps_out = 1
fe_pip.extract_features(selected_features=selected_features, target=None,
lags=None, rolling_windows=[4,12,24], drop_ogs=0)
X = fe_pip.process(n_steps, n_steps_out)
n_steps_predict = 72
X_input = X[-n_steps_predict:]
return X_input
# Function to automatically refresh weather data and predictions every 6 hours
@st.cache_data(ttl=6*3600) # Cache data for 6 hours
def auto_update_data_and_prediction():
# Download new weather data
raw_data = download_data()
if raw_data.empty:
st.error(" ⚠️ We are currently experiencing issues retrieving the latest weather data. Please try again later 🚀")
st.image("https://media.giphy.com/media/JIX9t2j0ZTN9S/giphy.gif", caption="Hang in there, we're working on it! 🐱")
return None, None, None, None
raw_data = raw_data.dropna()
data = preprocess_function(raw_data)
input = input_processing(data)
# Make prediction
predictionO3 = modelO3.predict(input)
predictionO3 = fitted_scalerO3_targets.inverse_transform(predictionO3)
predictionO3_flat = predictionO3.flatten() # Flatten to 1D array
rounded_predictionsO3 = [round(pred, 2) for pred in predictionO3_flat]
predictionNO2 = modelNO2.predict(input)
predictionNO2 = fitted_scalerNO2_targets.inverse_transform(predictionNO2)
predictionNO2_flat = predictionNO2.flatten() # Flatten to 1D array
rounded_predictionsNO2 = [round(pred, 2) for pred in predictionNO2_flat]
# Create DataFrame for predictions
hours = np.arange(1, len(predictionO3_flat) + 1)
prediction_df = pd.DataFrame({
"O₃ Prediction": rounded_predictionsO3,
"NO₂ Prediction": rounded_predictionsNO2
}, index=hours
)
# Adjust raw_data to match the length of prediction_df
raw_data = raw_data.iloc[-len(prediction_df):] # Slice to match length
prediction_df["Date"] = raw_data["date"].values # Add "date" column to prediction_df
# Save data and predictions
prediction_df.to_csv(f"data/prediction_{datetime.now().strftime('%Y-%m-%d_%H-%M')}.csv", index=False)
return raw_data, data, prediction_df, datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# Streamlit App Interface
st.title("O₃ and NO₂ Forecast Dashboard")
# Adding navigation using session state
if 'page' not in st.session_state:
st.session_state.page = 'dashboard'
def show_admin_login():
st.session_state.page = 'admin_login'
def show_dashboard():
st.session_state.page = 'dashboard'
def show_admin_page():
st.session_state.page = 'admin_page'
# Call auto-update function to refresh data and predictions every 6 hours
raw_data, hourly_dataframe, prediction_df, last_updated = auto_update_data_and_prediction()
if st.session_state.page == 'dashboard':
# Create a link in the upper-right corner for logging in as Admin
st.sidebar.write("Log in as Admin")
st.sidebar.button("Admin Login", on_click=show_admin_login)
st.markdown("### Welcome to the User Dashboard!")
st.markdown("This application allows to forecast O₃ and NO₂ for the next three days in the area of the city of Utrecht, the Netherlands. Based on the KNMI data obtained from the [public API](https://open-meteo.com/en/docs/knmi-api#latitude=52.11&longitude=5.1806&hourly=temperature_2m,dew_point_2m,wind_speed_10m,wind_direction_10m,wind_gusts_10m,sunshine_duration,global_tilted_irradiance&daily=weather_code).")
check_missing_values(hourly_dataframe)
check_for_extreme_values(hourly_dataframe[48:]) # skip first two days of history forecast
check_covariate_shift(hourly_dataframe[48:])
# Check if any of the key variables are None or invalid
if any(var is None for var in [raw_data, hourly_dataframe, prediction_df, last_updated]):
st.error("⚠️ Data is missing or incomplete. Unable to proceed with the forecast or visualization.")
st.stop() # This stops the execution of the code further down the page
else:
# Save the last update timestamp in session state after calling the function
if 'last_updated' not in st.session_state:
st.session_state['last_updated'] = last_updated
# Display last update timestamp
st.write(f"Last updated: {st.session_state['last_updated']}")
ozone_guideline = 100
no2_guideline = 25
st.markdown(f"""
### WHO Guidelines for Air Quality
- **O₃ (Ozone):** {ozone_guideline} µg/m³ (8-hour average)
- **NO₂ (Nitrogen Dioxide):** {no2_guideline} µg/m³ (24-hour average)
These thresholds represent the maximum recommended concentrations to ensure public health.
""")
#current_values = fetch_measurements()
#current_values
st.markdown("### Forecast")
#raw_data
# st.markdown("Below is the preprocessed data:")
# data_placeholder = st.empty()
# prediction_graph_placeholder = st.empty()
# data_placeholder.write(hourly_dataframe)
# Plot predictions as a graph
fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(12, 15))
# Calculate the rolling averages for O₃ (8-hour) and NO₂ (24-hour)
prediction_df['O₃ 8-hour Average'] = prediction_df['O₃ Prediction'].rolling(window=8).mean()
prediction_df['NO₂ 24-hour Average'] = prediction_df['NO₂ Prediction'].rolling(window=24).mean()
# Plot for O₃ with 8-hour rolling average
ax1.plot(prediction_df["Date"], prediction_df['O₃ Prediction'], marker='o', linestyle='-', color='b', label="O₃ Prediction")
ax1.plot(prediction_df["Date"], prediction_df['O₃ 8-hour Average'], marker='o', linestyle='-', color='orange', label="8-hour Average")
ax1.axhline(y=ozone_guideline, color='r', linestyle='--', label="Guideline") # WHO guideline
ax1.fill_between(prediction_df["Date"], ozone_guideline, prediction_df['O₃ 8-hour Average'],
where=(prediction_df['O₃ 8-hour Average'] > ozone_guideline), color='red', alpha=0.3)
ax1.set_xlabel('Hours', fontsize=14)
ax1.set_ylabel('O₃ Prediction (µg/m³)', fontsize=14)
ax1.set_title('O₃ Prediction and 8-hour Average', fontsize=16)
ax1.xaxis.set_major_formatter(mdates.DateFormatter('%d-%m %H')) # Change the format to DD-MM HH
ax1.legend(fontsize=12)
# Plot for NO₂ with 24-hour rolling average
ax2.plot(prediction_df["Date"], prediction_df['NO₂ Prediction'], marker='o', linestyle='-', color='g', label="NO₂ Prediction")
ax2.plot(prediction_df["Date"], prediction_df['NO₂ 24-hour Average'], marker='o', linestyle='-', color='purple', label="24-hour Average")
ax2.axhline(y=no2_guideline, color='r', linestyle='--', label="Guideline") # WHO guideline
ax2.fill_between(prediction_df["Date"], no2_guideline, prediction_df['NO₂ 24-hour Average'],
where=(prediction_df['NO₂ 24-hour Average'] > no2_guideline), color='red', alpha=0.3)
ax2.set_xlabel('Hours', fontsize=14)
ax2.set_ylabel('NO₂ Prediction (µg/m³)', fontsize=14)
ax2.set_title('NO₂ Prediction and 24-hour Average', fontsize=16)
ax2.xaxis.set_major_formatter(mdates.DateFormatter('%d-%m %H')) # Change the format to DD-MM HH
ax2.legend(fontsize=12)
# Show the combined plot
st.pyplot(fig)
# Option to display the table
if st.checkbox("Show prediction table", value=False ):
st.write("Predicted values:")
st.table(prediction_df)
elif st.session_state.page == 'admin_login':
st.sidebar.button("Back to Dashboard", on_click=show_dashboard)
st.markdown("### Admin Login")
st.markdown('For demonstration purposes, every combination of username and password is valid.')
username = st.text_input("Username")
password = st.text_input("Password", type="password")
# dummy login process
if username == "admin" and password == "admin":
st.success("Logged in as Admin")
else:
if username and password:
st.error("Invalid credentials, please try again.")
if username and password:
st.button("Login", on_click=show_admin_page)
else:
st.button("Login", disabled=True)
elif st.session_state.page == 'admin_page':
st.sidebar.success("You are logged in as Admin")
st.markdown("### Admin Page")
st.markdown("Welcome to the Admin Page. Here, you can manage predictions and settings.")
performance_df = pd.read_csv('results/performance.csv')
month_names = ["April", "July", "October", "December"]
month_map = {
"April" : 4,
"July" : 7,
"October" : 10,
"December" : 12
}
month_choice = st.sidebar.selectbox("Select a month to view predictions", month_names)
predictions_df = pd.read_csv(f'results/predictions{month_map[month_choice]}.csv')
model_choice = ['MLP']
model_data = performance_df[performance_df['model_name'] == model_choice[0]]
model_predictions = predictions_df[predictions_df['model_name'] == model_choice[0]]
history_dfs_O3 = {}
history_dfs_NO2 = {}
for model in model_choice:
history_dfs_O3[model] = pd.read_csv(f'results/history_{model}_O3.csv')
history_dfs_NO2[model] = pd.read_csv(f'results/history_{model}_NO2.csv')
# Display Model Performance
st.header("Model Monitoring Dashboard")
st.subheader(f"Current prediction model: {model_choice[0]}")
col1, col2, col3 = st.columns(3)
for i, model in enumerate(model_choice):
with [col1, col2, col3][i]:
st.subheader(f"Performance")
col_o3_1, col_o3_2 = st.columns(2)
with col_o3_1:
st.metric("MSE (O3)", f"{model_data.loc[model_data['model_name'] == model, 'mse_loss_O3'].values[0]:.4f}")
with col_o3_2:
st.metric("MAE (O3)", f"{model_data.loc[model_data['model_name'] == model, 'mae_loss_O3'].values[0]:.4f}")
# Second row for NO2 metrics
col_no2_1, col_no2_2 = st.columns(2)
with col_no2_1:
st.metric("MSE (NO2)", f"{model_data.loc[model_data['model_name'] == model, 'mse_loss_NO2'].values[0]:.4f}")
with col_no2_2:
st.metric("MAE (NO2)", f"{model_data.loc[model_data['model_name'] == model, 'mae_loss_NO2'].values[0]:.4f}")
# Plot for O3 predictions
st.subheader("O3 Predictions")
fig, ax = plt.subplots()
time_values = model_predictions['time'].unique()
actual_O3 = model_predictions.groupby('time')['actual_O3'].first().reindex(time_values)
# Select number of hours
num_hours = st.sidebar.slider("Select Number of Hours to Display", min_value=1, max_value=len(time_values), value=240)
time_values = time_values[:num_hours]
actual_O3 = actual_O3.reindex(time_values)
# Plot actual O3 values
ax.plot(time_values, actual_O3, label='Actual O3', color='yellowgreen', zorder=2)
for model in model_choice:
mdl_pred_filt = model_predictions[(model_predictions['model_name'] == model) &
(model_predictions['time'].isin(time_values))]
ax.plot(mdl_pred_filt['time'], mdl_pred_filt['predicted_O3'],
linestyle='--', label=f'{model}', color='navy')
plt.title("Actual and Predicted Values for O3")
ax.set_xlabel('Time (hours)')
ax.set_ylabel('O3 Levels')
ax.legend()
st.pyplot(fig)
plt.clf()
# Plot for NO2
st.subheader("NO2 Predictions")
fig, ax = plt.subplots()
time_values = model_predictions['time'].unique()
actual_NO2 = model_predictions.groupby('time')['actual_NO2'].first().reindex(time_values)
time_values = time_values[:num_hours]
actual_NO2 = actual_NO2.reindex(time_values)
ax.plot(time_values, actual_NO2, label='Actual', color='yellowgreen')
for model in model_choice:
mdl_pred_filt = model_predictions[(model_predictions['model_name'] == model) &
(model_predictions['time'].isin(time_values))]
ax.plot(mdl_pred_filt['time'], mdl_pred_filt['predicted_NO2'],
linestyle='--', label=f'{model}', color='navy')
plt.title("Actual and Predicted Values for NO2")
ax.set_xlabel('Time (hours)')
ax.set_ylabel('NO2 Levels')
ax.legend()
st.pyplot(fig)
plt.clf()
# Plot: Performance Over Time for O3
st.subheader("Model Performance Over Time (O3)")
fig, ax = plt.subplots()
for model in model_choice:
ax.plot(history_dfs_O3[model]["loss"], label=f"{model} Training Loss", color='navy')
ax.plot(history_dfs_O3[model]["val_loss"], linestyle='--', label=f"{model} Validation Loss", color='navy')
ax.set_title("Training and Validation Loss for O3")
ax.set_xlabel("Epochs")
ax.set_ylabel("Loss")
ax.legend()
st.pyplot(fig)
plt.clf()
# Performance Over Time for NO2
st.subheader("Model Performance Over Time (NO2)")
fig, ax = plt.subplots()
for model in model_choice:
ax.plot(history_dfs_NO2[model]["loss"], label=f"{model} Training Loss", color='navy')
ax.plot(history_dfs_NO2[model]["val_loss"], linestyle='--', label=f"{model} Validation Loss", color='navy')
ax.set_title("Training and Validation Loss for NO2")
ax.set_xlabel("Epochs")
ax.set_ylabel("Loss")
ax.legend()
st.pyplot(fig)
plt.clf()
st.sidebar.button("Back to Dashboard", on_click=show_dashboard)
# Insert a link on the admin page
st.markdown("[Go to data management page](https://www.youtube.com/watch?v=8eJZQbT0fgc)", unsafe_allow_html=True)