Spaces:
Sleeping
Sleeping
Download app.py from MLINPrediction/pollution_prediction: direct link, hf CLI and curl.
- Browser
- Download file 15.5 kB
-
https://huggingface.co/spaces/MLINPrediction/pollution_prediction/resolve/main/app.py
- Command line
-
hf download hf://spaces/MLINPrediction/pollution_prediction/app.py
-
curl -L -o app.py https://huggingface.co/spaces/MLINPrediction/pollution_prediction/resolve/main/app.py
15.5 kB
| import streamlit as st | |
| import pandas as pd | |
| import tensorflow as tf | |
| from huggingface_hub import hf_hub_download | |
| import numpy as np | |
| import openmeteo_requests | |
| import requests_cache | |
| from retry_requests import retry | |
| import pickle as pk | |
| import matplotlib.pyplot as plt | |
| from datetime import datetime, timedelta | |
| import matplotlib.dates as mdates | |
| import requests | |
| import logging | |
| from preprocessing.pr_featureEng import FeatureEngineeringPredict | |
| from preprocessing.forecast_dataset import preprocess_function, check_missing_values, check_for_extreme_values, check_covariate_shift | |
| from preprocessing.forecast_api import download_data | |
| # Model is cached to prevent reloading | |
| def load_model(model_name): | |
| model_dir = hf_hub_download(repo_id="senyakk/pollution_MLP", filename=model_name) | |
| return tf.keras.models.load_model(model_dir) | |
| def load_pca_scalers(): | |
| fitted_pca = pk.load(open("preprocessing/pca.pkl",'rb')) | |
| fitted_scaler = pk.load(open("preprocessing/scaler_features.pkl",'rb')) | |
| fitted_scalerO3_targets = pk.load(open("preprocessing/scalerO3_targets.pkl",'rb')) | |
| fitted_scalerNO2_targets = pk.load(open("preprocessing/scalerNO2_targets.pkl",'rb')) | |
| return fitted_pca, fitted_scaler, fitted_scalerO3_targets, fitted_scalerNO2_targets | |
| modelNO2 = load_model("mlpNO2.keras") | |
| modelO3 = load_model("mlpO3.keras") | |
| fitted_pca, fitted_scaler, fitted_scalerO3_targets, fitted_scalerNO2_targets = load_pca_scalers() | |
| def input_processing(data): | |
| fe_pip = FeatureEngineeringPredict(data, fitted_pca, fitted_scaler) | |
| selected_features = ['MWS', 'WG', 'T', 'TD', 'SD', 'GR'] | |
| n_steps = 3 | |
| n_steps_out = 1 | |
| fe_pip.extract_features(selected_features=selected_features, target=None, | |
| lags=None, rolling_windows=[4,12,24], drop_ogs=0) | |
| X = fe_pip.process(n_steps, n_steps_out) | |
| n_steps_predict = 72 | |
| X_input = X[-n_steps_predict:] | |
| return X_input | |
| # Function to automatically refresh weather data and predictions every 6 hours | |
| # Cache data for 6 hours | |
| def auto_update_data_and_prediction(): | |
| # Download new weather data | |
| raw_data = download_data() | |
| if raw_data.empty: | |
| st.error(" ⚠️ We are currently experiencing issues retrieving the latest weather data. Please try again later 🚀") | |
| st.image("https://media.giphy.com/media/JIX9t2j0ZTN9S/giphy.gif", caption="Hang in there, we're working on it! 🐱") | |
| return None, None, None, None | |
| raw_data = raw_data.dropna() | |
| data = preprocess_function(raw_data) | |
| input = input_processing(data) | |
| # Make prediction | |
| predictionO3 = modelO3.predict(input) | |
| predictionO3 = fitted_scalerO3_targets.inverse_transform(predictionO3) | |
| predictionO3_flat = predictionO3.flatten() # Flatten to 1D array | |
| rounded_predictionsO3 = [round(pred, 2) for pred in predictionO3_flat] | |
| predictionNO2 = modelNO2.predict(input) | |
| predictionNO2 = fitted_scalerNO2_targets.inverse_transform(predictionNO2) | |
| predictionNO2_flat = predictionNO2.flatten() # Flatten to 1D array | |
| rounded_predictionsNO2 = [round(pred, 2) for pred in predictionNO2_flat] | |
| # Create DataFrame for predictions | |
| hours = np.arange(1, len(predictionO3_flat) + 1) | |
| prediction_df = pd.DataFrame({ | |
| "O₃ Prediction": rounded_predictionsO3, | |
| "NO₂ Prediction": rounded_predictionsNO2 | |
| }, index=hours | |
| ) | |
| # Adjust raw_data to match the length of prediction_df | |
| raw_data = raw_data.iloc[-len(prediction_df):] # Slice to match length | |
| prediction_df["Date"] = raw_data["date"].values # Add "date" column to prediction_df | |
| # Save data and predictions | |
| prediction_df.to_csv(f"data/prediction_{datetime.now().strftime('%Y-%m-%d_%H-%M')}.csv", index=False) | |
| return raw_data, data, prediction_df, datetime.now().strftime('%Y-%m-%d %H:%M:%S') | |
| # Streamlit App Interface | |
| st.title("O₃ and NO₂ Forecast Dashboard") | |
| # Adding navigation using session state | |
| if 'page' not in st.session_state: | |
| st.session_state.page = 'dashboard' | |
| def show_admin_login(): | |
| st.session_state.page = 'admin_login' | |
| def show_dashboard(): | |
| st.session_state.page = 'dashboard' | |
| def show_admin_page(): | |
| st.session_state.page = 'admin_page' | |
| # Call auto-update function to refresh data and predictions every 6 hours | |
| raw_data, hourly_dataframe, prediction_df, last_updated = auto_update_data_and_prediction() | |
| if st.session_state.page == 'dashboard': | |
| # Create a link in the upper-right corner for logging in as Admin | |
| st.sidebar.write("Log in as Admin") | |
| st.sidebar.button("Admin Login", on_click=show_admin_login) | |
| st.markdown("### Welcome to the User Dashboard!") | |
| st.markdown("This application allows to forecast O₃ and NO₂ for the next three days in the area of the city of Utrecht, the Netherlands. Based on the KNMI data obtained from the [public API](https://open-meteo.com/en/docs/knmi-api#latitude=52.11&longitude=5.1806&hourly=temperature_2m,dew_point_2m,wind_speed_10m,wind_direction_10m,wind_gusts_10m,sunshine_duration,global_tilted_irradiance&daily=weather_code).") | |
| check_missing_values(hourly_dataframe) | |
| check_for_extreme_values(hourly_dataframe[48:]) # skip first two days of history forecast | |
| check_covariate_shift(hourly_dataframe[48:]) | |
| # Check if any of the key variables are None or invalid | |
| if any(var is None for var in [raw_data, hourly_dataframe, prediction_df, last_updated]): | |
| st.error("⚠️ Data is missing or incomplete. Unable to proceed with the forecast or visualization.") | |
| st.stop() # This stops the execution of the code further down the page | |
| else: | |
| # Save the last update timestamp in session state after calling the function | |
| if 'last_updated' not in st.session_state: | |
| st.session_state['last_updated'] = last_updated | |
| # Display last update timestamp | |
| st.write(f"Last updated: {st.session_state['last_updated']}") | |
| ozone_guideline = 100 | |
| no2_guideline = 25 | |
| st.markdown(f""" | |
| ### WHO Guidelines for Air Quality | |
| - **O₃ (Ozone):** {ozone_guideline} µg/m³ (8-hour average) | |
| - **NO₂ (Nitrogen Dioxide):** {no2_guideline} µg/m³ (24-hour average) | |
| These thresholds represent the maximum recommended concentrations to ensure public health. | |
| """) | |
| #current_values = fetch_measurements() | |
| #current_values | |
| st.markdown("### Forecast") | |
| #raw_data | |
| # st.markdown("Below is the preprocessed data:") | |
| # data_placeholder = st.empty() | |
| # prediction_graph_placeholder = st.empty() | |
| # data_placeholder.write(hourly_dataframe) | |
| # Plot predictions as a graph | |
| fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(12, 15)) | |
| # Calculate the rolling averages for O₃ (8-hour) and NO₂ (24-hour) | |
| prediction_df['O₃ 8-hour Average'] = prediction_df['O₃ Prediction'].rolling(window=8).mean() | |
| prediction_df['NO₂ 24-hour Average'] = prediction_df['NO₂ Prediction'].rolling(window=24).mean() | |
| # Plot for O₃ with 8-hour rolling average | |
| ax1.plot(prediction_df["Date"], prediction_df['O₃ Prediction'], marker='o', linestyle='-', color='b', label="O₃ Prediction") | |
| ax1.plot(prediction_df["Date"], prediction_df['O₃ 8-hour Average'], marker='o', linestyle='-', color='orange', label="8-hour Average") | |
| ax1.axhline(y=ozone_guideline, color='r', linestyle='--', label="Guideline") # WHO guideline | |
| ax1.fill_between(prediction_df["Date"], ozone_guideline, prediction_df['O₃ 8-hour Average'], | |
| where=(prediction_df['O₃ 8-hour Average'] > ozone_guideline), color='red', alpha=0.3) | |
| ax1.set_xlabel('Hours', fontsize=14) | |
| ax1.set_ylabel('O₃ Prediction (µg/m³)', fontsize=14) | |
| ax1.set_title('O₃ Prediction and 8-hour Average', fontsize=16) | |
| ax1.xaxis.set_major_formatter(mdates.DateFormatter('%d-%m %H')) # Change the format to DD-MM HH | |
| ax1.legend(fontsize=12) | |
| # Plot for NO₂ with 24-hour rolling average | |
| ax2.plot(prediction_df["Date"], prediction_df['NO₂ Prediction'], marker='o', linestyle='-', color='g', label="NO₂ Prediction") | |
| ax2.plot(prediction_df["Date"], prediction_df['NO₂ 24-hour Average'], marker='o', linestyle='-', color='purple', label="24-hour Average") | |
| ax2.axhline(y=no2_guideline, color='r', linestyle='--', label="Guideline") # WHO guideline | |
| ax2.fill_between(prediction_df["Date"], no2_guideline, prediction_df['NO₂ 24-hour Average'], | |
| where=(prediction_df['NO₂ 24-hour Average'] > no2_guideline), color='red', alpha=0.3) | |
| ax2.set_xlabel('Hours', fontsize=14) | |
| ax2.set_ylabel('NO₂ Prediction (µg/m³)', fontsize=14) | |
| ax2.set_title('NO₂ Prediction and 24-hour Average', fontsize=16) | |
| ax2.xaxis.set_major_formatter(mdates.DateFormatter('%d-%m %H')) # Change the format to DD-MM HH | |
| ax2.legend(fontsize=12) | |
| # Show the combined plot | |
| st.pyplot(fig) | |
| # Option to display the table | |
| if st.checkbox("Show prediction table", value=False ): | |
| st.write("Predicted values:") | |
| st.table(prediction_df) | |
| elif st.session_state.page == 'admin_login': | |
| st.sidebar.button("Back to Dashboard", on_click=show_dashboard) | |
| st.markdown("### Admin Login") | |
| st.markdown('For demonstration purposes, every combination of username and password is valid.') | |
| username = st.text_input("Username") | |
| password = st.text_input("Password", type="password") | |
| # dummy login process | |
| if username == "admin" and password == "admin": | |
| st.success("Logged in as Admin") | |
| else: | |
| if username and password: | |
| st.error("Invalid credentials, please try again.") | |
| if username and password: | |
| st.button("Login", on_click=show_admin_page) | |
| else: | |
| st.button("Login", disabled=True) | |
| elif st.session_state.page == 'admin_page': | |
| st.sidebar.success("You are logged in as Admin") | |
| st.markdown("### Admin Page") | |
| st.markdown("Welcome to the Admin Page. Here, you can manage predictions and settings.") | |
| performance_df = pd.read_csv('results/performance.csv') | |
| month_names = ["April", "July", "October", "December"] | |
| month_map = { | |
| "April" : 4, | |
| "July" : 7, | |
| "October" : 10, | |
| "December" : 12 | |
| } | |
| month_choice = st.sidebar.selectbox("Select a month to view predictions", month_names) | |
| predictions_df = pd.read_csv(f'results/predictions{month_map[month_choice]}.csv') | |
| model_choice = ['MLP'] | |
| model_data = performance_df[performance_df['model_name'] == model_choice[0]] | |
| model_predictions = predictions_df[predictions_df['model_name'] == model_choice[0]] | |
| history_dfs_O3 = {} | |
| history_dfs_NO2 = {} | |
| for model in model_choice: | |
| history_dfs_O3[model] = pd.read_csv(f'results/history_{model}_O3.csv') | |
| history_dfs_NO2[model] = pd.read_csv(f'results/history_{model}_NO2.csv') | |
| # Display Model Performance | |
| st.header("Model Monitoring Dashboard") | |
| st.subheader(f"Current prediction model: {model_choice[0]}") | |
| col1, col2, col3 = st.columns(3) | |
| for i, model in enumerate(model_choice): | |
| with [col1, col2, col3][i]: | |
| st.subheader(f"Performance") | |
| col_o3_1, col_o3_2 = st.columns(2) | |
| with col_o3_1: | |
| st.metric("MSE (O3)", f"{model_data.loc[model_data['model_name'] == model, 'mse_loss_O3'].values[0]:.4f}") | |
| with col_o3_2: | |
| st.metric("MAE (O3)", f"{model_data.loc[model_data['model_name'] == model, 'mae_loss_O3'].values[0]:.4f}") | |
| # Second row for NO2 metrics | |
| col_no2_1, col_no2_2 = st.columns(2) | |
| with col_no2_1: | |
| st.metric("MSE (NO2)", f"{model_data.loc[model_data['model_name'] == model, 'mse_loss_NO2'].values[0]:.4f}") | |
| with col_no2_2: | |
| st.metric("MAE (NO2)", f"{model_data.loc[model_data['model_name'] == model, 'mae_loss_NO2'].values[0]:.4f}") | |
| # Plot for O3 predictions | |
| st.subheader("O3 Predictions") | |
| fig, ax = plt.subplots() | |
| time_values = model_predictions['time'].unique() | |
| actual_O3 = model_predictions.groupby('time')['actual_O3'].first().reindex(time_values) | |
| # Select number of hours | |
| num_hours = st.sidebar.slider("Select Number of Hours to Display", min_value=1, max_value=len(time_values), value=240) | |
| time_values = time_values[:num_hours] | |
| actual_O3 = actual_O3.reindex(time_values) | |
| # Plot actual O3 values | |
| ax.plot(time_values, actual_O3, label='Actual O3', color='yellowgreen', zorder=2) | |
| for model in model_choice: | |
| mdl_pred_filt = model_predictions[(model_predictions['model_name'] == model) & | |
| (model_predictions['time'].isin(time_values))] | |
| ax.plot(mdl_pred_filt['time'], mdl_pred_filt['predicted_O3'], | |
| linestyle='--', label=f'{model}', color='navy') | |
| plt.title("Actual and Predicted Values for O3") | |
| ax.set_xlabel('Time (hours)') | |
| ax.set_ylabel('O3 Levels') | |
| ax.legend() | |
| st.pyplot(fig) | |
| plt.clf() | |
| # Plot for NO2 | |
| st.subheader("NO2 Predictions") | |
| fig, ax = plt.subplots() | |
| time_values = model_predictions['time'].unique() | |
| actual_NO2 = model_predictions.groupby('time')['actual_NO2'].first().reindex(time_values) | |
| time_values = time_values[:num_hours] | |
| actual_NO2 = actual_NO2.reindex(time_values) | |
| ax.plot(time_values, actual_NO2, label='Actual', color='yellowgreen') | |
| for model in model_choice: | |
| mdl_pred_filt = model_predictions[(model_predictions['model_name'] == model) & | |
| (model_predictions['time'].isin(time_values))] | |
| ax.plot(mdl_pred_filt['time'], mdl_pred_filt['predicted_NO2'], | |
| linestyle='--', label=f'{model}', color='navy') | |
| plt.title("Actual and Predicted Values for NO2") | |
| ax.set_xlabel('Time (hours)') | |
| ax.set_ylabel('NO2 Levels') | |
| ax.legend() | |
| st.pyplot(fig) | |
| plt.clf() | |
| # Plot: Performance Over Time for O3 | |
| st.subheader("Model Performance Over Time (O3)") | |
| fig, ax = plt.subplots() | |
| for model in model_choice: | |
| ax.plot(history_dfs_O3[model]["loss"], label=f"{model} Training Loss", color='navy') | |
| ax.plot(history_dfs_O3[model]["val_loss"], linestyle='--', label=f"{model} Validation Loss", color='navy') | |
| ax.set_title("Training and Validation Loss for O3") | |
| ax.set_xlabel("Epochs") | |
| ax.set_ylabel("Loss") | |
| ax.legend() | |
| st.pyplot(fig) | |
| plt.clf() | |
| # Performance Over Time for NO2 | |
| st.subheader("Model Performance Over Time (NO2)") | |
| fig, ax = plt.subplots() | |
| for model in model_choice: | |
| ax.plot(history_dfs_NO2[model]["loss"], label=f"{model} Training Loss", color='navy') | |
| ax.plot(history_dfs_NO2[model]["val_loss"], linestyle='--', label=f"{model} Validation Loss", color='navy') | |
| ax.set_title("Training and Validation Loss for NO2") | |
| ax.set_xlabel("Epochs") | |
| ax.set_ylabel("Loss") | |
| ax.legend() | |
| st.pyplot(fig) | |
| plt.clf() | |
| st.sidebar.button("Back to Dashboard", on_click=show_dashboard) | |
| # Insert a link on the admin page | |
| st.markdown("[Go to data management page](https://www.youtube.com/watch?v=8eJZQbT0fgc)", unsafe_allow_html=True) | |