project / app.py
dsid271's picture
Cache openmeteo responses
cab6615
Raw History Blame Contribute Delete
33 kB
import os
from typing import List, Optional, Tuple
# Set environment variables for TensorFlow/Keras backend
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
BACKEND = 'tensorflow'
os.environ['KERAS_BACKEND'] = BACKEND
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import load_model
from tensorflow.keras.utils import custom_object_scope
import requests
import pandas as pd
from datetime import datetime, timedelta
import pytz
import json
import traceback
import tempfile
import time
from threading import Lock
import firebase_admin
from firebase_admin import credentials, db
# --- FIX 1: Fail loud on missing TKAN. No zero-returning stub. ---
TKAN_IMPORT_OK = False
TKAN_IMPORT_ERROR = None
TKANLayer = None
try:
from tkan import TKAN as TKANLayer
TKAN_IMPORT_OK = True
print("TKAN imported from package 'tkan'.")
except ImportError:
try:
from keras_efficient_kan import TKANLayer
TKAN_IMPORT_OK = True
print("TKAN imported from package 'keras_efficient_kan'.")
except ImportError as e:
TKAN_IMPORT_ERROR = e
print("=" * 70)
print("CRITICAL: Real TKAN layer could not be imported.")
print("Neither 'tkan' nor 'keras_efficient_kan' is installed.")
print("Refusing to serve until the correct package is installed.")
print(f"Underlying import error: {e}")
print("=" * 70)
# Hard fail: never boot with a missing TKAN layer (would otherwise serve zeros).
if not TKAN_IMPORT_OK:
raise ImportError(
"Real TKAN layer unavailable ΓÇö aborting startup to avoid serving "
f"zero predictions. Install 'tkan' or 'keras_efficient_kan'. "
f"Original error: {TKAN_IMPORT_ERROR}"
)
# Constants
MAX_N_AHEAD = 168 # Max 7 days of predictions
SEQUENCE_LENGTH_MIN = 24 # Minimum historical data required
ROLLING_WINDOW = 24 * 14 # 336h rolling-median window (CONFIRMED from training notebook)
# Initialize Firebase using environment variables
import tempfile
firebase_credential_json = os.environ.get("FIREBASE_CREDENTIAL_JSON")
firebase_initialized = False
if firebase_credential_json:
try:
temp_dir = tempfile.gettempdir()
firebase_credential_path = os.path.join(temp_dir, "firebase-credentials.json")
with open(firebase_credential_path, "w", encoding="utf-8") as f:
f.write(firebase_credential_json)
print(f"Wrote Firebase credentials to writable temp path: {firebase_credential_path}")
FIREBASE_DATABASE_URL = os.getenv("FIREBASE_DATABASE_URL")
if not os.path.exists(firebase_credential_path):
raise FileNotFoundError(f"Firebase credentials file not found at {firebase_credential_path}")
cred = credentials.Certificate(firebase_credential_path)
firebase_admin.initialize_app(cred, {'databaseURL': FIREBASE_DATABASE_URL})
firebase_initialized = True
print("Firebase initialized successfully")
except Exception as e:
print(f"Firebase initialization failed: {e}")
firebase_initialized = False
else:
print("Firebase credential not found in environment variables. Will use fallback (Open-Meteo only).")
OPEN_METEO_CACHE_TTL_SECONDS = int(os.getenv("OPEN_METEO_CACHE_TTL_SECONDS", "900"))
OPEN_METEO_CACHE_MAX_ENTRIES = int(os.getenv("OPEN_METEO_CACHE_MAX_ENTRIES", "64"))
open_meteo_cache = {}
open_meteo_cache_lock = Lock()
def _normalize_params(params):
normalized = []
for key, value in params.items():
if isinstance(value, list):
normalized.append((key, tuple(value)))
elif isinstance(value, dict):
normalized.append((key, tuple(sorted(value.items()))))
else:
normalized.append((key, value))
return tuple(sorted(normalized))
def _make_open_meteo_cache_key(url, params):
return (url, _normalize_params(params))
def _fetch_json_with_retry(url, params, max_attempts=3, backoff_factor=1.0):
attempt = 1
while True:
response = requests.get(url, params=params, timeout=30)
if response.status_code == 429 or 500 <= response.status_code < 600:
if attempt >= max_attempts:
response.raise_for_status()
sleep_seconds = backoff_factor * (2 ** (attempt - 1))
print(f"Open-Meteo request returned {response.status_code}. Retrying in {sleep_seconds:.1f}s (attempt {attempt}/{max_attempts})")
time.sleep(sleep_seconds)
attempt += 1
continue
response.raise_for_status()
return response.json()
def get_open_meteo_cached_json(url, params):
key = _make_open_meteo_cache_key(url, params)
now = datetime.now(pytz.utc)
with open_meteo_cache_lock:
cached = open_meteo_cache.get(key)
if cached and (now - cached["ts"]).total_seconds() < OPEN_METEO_CACHE_TTL_SECONDS:
print(f"Open-Meteo cache hit for {url}")
return cached["data"]
data = _fetch_json_with_retry(url, params)
with open_meteo_cache_lock:
if len(open_meteo_cache) >= OPEN_METEO_CACHE_MAX_ENTRIES:
oldest_key = min(open_meteo_cache.items(), key=lambda pair: pair[1]["ts"])[0]
del open_meteo_cache[oldest_key]
open_meteo_cache[key] = {"ts": now, "data": data}
print(f"Open-Meteo cache set for {url}")
return data
class MinMaxScaler:
def __init__(self, feature_axis=None, minmax_range=(0, 1)):
self.feature_axis = feature_axis
self.min_ = None
self.max_ = None
self.scale_ = None
self.minmax_range = minmax_range
def load_attributes(self, attributes):
self.min_ = np.array(attributes['min_']) if isinstance(attributes['min_'], list) else attributes['min_']
self.max_ = np.array(attributes['max_']) if isinstance(attributes['max_'], list) else attributes['max_']
self.scale_ = np.array(attributes['scale_']) if isinstance(attributes['scale_'], list) else attributes['scale_']
self.minmax_range = tuple(attributes['minmax_range']) if isinstance(attributes['minmax_range'], list) else attributes['minmax_range']
# --- FIX 2: guard against zero scale (constant feature -> inf/nan). ---
self.scale_ = np.where(np.asarray(self.scale_) == 0, 1, self.scale_)
def fit(self, X):
if X.ndim == 3 and self.feature_axis is not None:
axis = tuple(i for i in range(X.ndim) if i != self.feature_axis)
self.min_ = np.min(X, axis=axis)
self.max_ = np.max(X, axis=axis)
elif X.ndim == 2:
self.min_ = np.min(X, axis=0)
self.max_ = np.max(X, axis=0)
elif X.ndim == 1:
self.min_ = np.min(X)
self.max_ = np.max(X)
else:
raise ValueError("Data must be 1D, 2D, or 3D.")
self.scale_ = self.max_ - self.min_
self.scale_ = np.where(np.asarray(self.scale_) == 0, 1, self.scale_)
return self
def transform(self, X):
if self.min_ is None or self.max_ is None or self.scale_ is None:
raise ValueError("Scaler attributes not loaded or scaler not fitted.")
X_scaled = (X - self.min_) / self.scale_
X_scaled = X_scaled * (self.minmax_range[1] - self.minmax_range[0]) + self.minmax_range[0]
return X_scaled
def inverse_transform(self, X_scaled):
if self.min_ is None or self.max_ is None or self.scale_ is None:
raise ValueError("Scaler attributes not loaded or scaler not fitted.")
X = (X_scaled - self.minmax_range[0]) / (self.minmax_range[1] - self.minmax_range[0])
X = X * self.scale_ + self.min_
return X
# --- AQI breakpoints and calculation functions ---
# NOTE (confirmed caveat): CO values in the training data range 106-1869, far above
# the top CO breakpoint of 17.0, so CO saturates. This unit mismatch is baked into
# training; serving computes AQI identically so it stays self-consistent, but the
# predicted "AQI" is NOT standards-compliant. Left unchanged deliberately.
aqi_breakpoints = {
'pm25': [(0, 50, 0, 50), (51, 100, 51, 100), (101, 200, 101, 200), (201, 300, 201, 300)],
'pm10': [(0, 50, 0, 50), (51, 100, 51, 100), (101, 250, 101, 200), (251, 350, 201, 300)],
'co': [(0, 1.0, 0, 50), (1.1, 2.0, 51, 100), (2.1, 10.0, 101, 200), (10.1, 17.0, 201, 300)]
}
def calculate_sub_aqi(concentration, breakpoints):
for i_low, i_high, c_low, c_high in breakpoints:
if c_low <= concentration <= c_high:
if c_high == c_low:
return i_low
return ((i_high - i_low) / (c_high - c_low)) * (concentration - c_low) + i_low
if concentration < breakpoints[0][2]:
return breakpoints[0][0]
elif concentration > breakpoints[-1][3]:
return breakpoints[-1][1]
else:
return np.nan
def calculate_overall_aqi(row, aqi_breakpoints):
sub_aqis = []
pollutant_mapping = {
'pm25': 'pm25',
'pm10': 'pm10',
'co': 'co',
'pm2_5': 'pm25',
'carbon_monoxide': 'co',
}
for api_pollutant, internal_pollutant in pollutant_mapping.items():
if api_pollutant in row:
concentration = row[api_pollutant]
if not pd.isna(concentration):
sub_aqi = calculate_sub_aqi(concentration, aqi_breakpoints.get(internal_pollutant, []))
sub_aqis.append(sub_aqi)
else:
sub_aqis.append(np.nan)
else:
sub_aqis.append(np.nan)
if sub_aqis and not all(pd.isna(sub_aqis)):
return np.nanmax(sub_aqis)
else:
return np.nan
def _build_ratio_sequence_from_processed(df_processed, sequence_length, n_ahead,
window_start_time_ts, window_end_time_ts,
required_columns):
"""
Shared ratio-transform pipeline used by BOTH Firebase and Open-Meteo paths.
df_processed: hourly, absolute-valued, columns == required_columns, tz-aware UTC index.
Returns (ratio_seq (1,seq,5), timestamps, future_exogenous_ratio (n_ahead,5),
denom_matrix (n_ahead,5)) or (None, None, None, None) on failure.
Mirrors training generate_data(): scaler_df = df.shift(n_ahead).rolling(336).median();
tmp_df = df / scaler_df.
"""
scaler_df_full = df_processed[required_columns].shift(n_ahead).rolling(ROLLING_WINDOW).median()
ratio_df_full = df_processed[required_columns] / scaler_df_full
rolling_median_aqi = scaler_df_full['calculated_aqi']
finite_median = rolling_median_aqi.dropna()
print(f"Rolling-median series computed. Finite points: {len(finite_median)}")
finite_scaler_rows = scaler_df_full.dropna()
if len(finite_scaler_rows):
fallback_denom_row = finite_scaler_rows.iloc[-1].values.astype(np.float64)
else:
fallback_denom_row = np.ones((len(required_columns),), dtype=np.float64)
fallback_denom_row = np.where(fallback_denom_row != 0, fallback_denom_row, 1.0)
# Recent absolute window (timestamps + last-row anchor + real forecast exogenous).
abs_recent_all = df_processed[
(df_processed.index >= window_start_time_ts) & (df_processed.index <= window_end_time_ts)
].copy()
abs_recent_all.dropna(inplace=True)
print(f"Shape after dropna on recent window: {abs_recent_all.shape}")
if len(abs_recent_all) < sequence_length:
print(f"Error: Only {len(abs_recent_all)} valid points, need {sequence_length}.")
return None, None, None, None
abs_recent_df = abs_recent_all.tail(sequence_length).copy()
# Ratio window = model input. NaN guard: 336h median must be finite across all steps.
ratio_recent = ratio_df_full.reindex(abs_recent_df.index)
if ratio_recent.isna().any().any():
print("Error: ratio window contains NaN ΓÇö 336h rolling median not finite across "
"all input steps. Not enough clean history.")
return None, None, None, None
latest_ratio_df = ratio_recent.tail(sequence_length).copy()
ratio_seq = latest_ratio_df.values.reshape(1, sequence_length, len(required_columns))
timestamps = latest_ratio_df.index.tolist()
last_ts = abs_recent_df.index[-1]
df_future = df_processed[df_processed.index > window_end_time_ts].copy()
df_future.dropna(inplace=True)
num_features = len(required_columns)
future_exogenous_ratio = np.zeros((n_ahead, num_features), dtype=np.float64)
denom_matrix = np.zeros((n_ahead, num_features), dtype=np.float64)
last_known_abs = abs_recent_df.iloc[-1].values.astype(np.float64)
for i in range(n_ahead):
pred_ts = last_ts + pd.Timedelta(hours=i + 1)
if pred_ts in scaler_df_full.index and not scaler_df_full.loc[pred_ts].isna().any():
denom_row = scaler_df_full.loc[pred_ts].values.astype(np.float64)
else:
denom_row = fallback_denom_row.copy()
denom_row = np.where(denom_row != 0, denom_row, 1.0)
denom_matrix[i] = denom_row
exo_abs = df_future.iloc[i].values.astype(np.float64) if i < len(df_future) else last_known_abs
future_exogenous_ratio[i] = exo_abs / denom_row
return ratio_seq, timestamps, future_exogenous_ratio, denom_matrix
# --- Function to retrieve data from Firebase ---
def get_firebase_data(sequence_length: int, n_ahead: int, latitude: float, longitude: float):
"""
Fetch enough Firebase history to compute the 336h rolling median, resample hourly,
compute calculated_aqi, and return an absolute-valued df_processed (or None).
Schema assumed from prior code: key 'datetime' as '%Y-%m-%dT%H:%M', pollutant keys
pm2_5 / pm10 / carbon_monoxide / temp. Adjust here if your DB differs.
"""
if not firebase_initialized:
return None
try:
print("Attempting to retrieve data from Firebase RTDB...")
ref = db.reference('/AQIData')
current_utc_time = datetime.now(pytz.utc)
# Need 336h + input window + margin, same as the Open-Meteo path.
fetch_hours = sequence_length + ROLLING_WINDOW + n_ahead + 72
window_start = current_utc_time - timedelta(hours=fetch_hours)
window_end = current_utc_time
firebase_data = ref.order_by_child('datetime').start_at(
window_start.strftime('%Y-%m-%dT%H:%M')
).end_at(window_end.strftime('%Y-%m-%dT%H:%M')).get()
if not firebase_data:
print("No data found in Firebase for the specified time range")
return None
data_list = []
for key, sensor_data in firebase_data.items():
try:
datetime_str = sensor_data.get('datetime')
if isinstance(datetime_str, str):
ts = pytz.utc.localize(datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M'))
data_list.append({
'time': ts,
'pm25': sensor_data.get('pm2_5', np.nan),
'pm10': sensor_data.get('pm10', np.nan),
'co': sensor_data.get('carbon_monoxide', np.nan),
'temp': sensor_data.get('temp', np.nan),
})
else:
print(f"Warning: item {key} missing/invalid 'datetime': {datetime_str}")
except ValueError as ve:
print(f"Error parsing datetime '{datetime_str}' for key {key}: {ve}")
continue
except Exception as e:
print(f"Unexpected error processing item {key}: {e}")
continue
if not data_list:
print("No valid data points parsed from Firebase.")
return None
df = pd.DataFrame(data_list).set_index('time').sort_index()
print(f"Retrieved {len(df)} data points from Firebase")
df_hourly = df.resample('h').mean()
# Gap check kept from your original logic.
time_diffs = df_hourly.index.to_series().diff()
max_gap = time_diffs.max()
if pd.notna(max_gap) and max_gap > timedelta(hours=1, minutes=30):
print(f"Data has gaps larger than 1.5 hours. Max gap: {max_gap}")
return None
df_hourly = df_hourly.ffill().bfill()
df_hourly['calculated_aqi'] = df_hourly.apply(
lambda row: calculate_overall_aqi(row, aqi_breakpoints), axis=1
)
required_columns = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co']
for col in required_columns:
if col not in df_hourly.columns:
df_hourly[col] = np.nan
df_processed = df_hourly[required_columns].copy()
if df_processed.index.tz is None:
df_processed.index = df_processed.index.tz_localize('UTC')
# Need at least 336h + window to have any finite median at all.
if len(df_processed) < ROLLING_WINDOW + sequence_length:
print(f"Firebase returned {len(df_processed)} hourly points; need "
f"{ROLLING_WINDOW + sequence_length} for a valid rolling median. "
f"Falling back to Open-Meteo.")
return None
print(f"Firebase df_processed ready. Shape: {df_processed.shape}")
return df_processed
except Exception as e:
print(f"Error retrieving data from Firebase: {e}")
traceback.print_exc()
return None
# --- Data retrieval function ---
# Returns: (ratio_input_sequence, timestamps, future_exogenous_RATIO, denom_matrix)
# ratio_input_sequence : (1, seq_len, 5) RATIO domain ΓÇö what the model consumes
# future_exogenous_RATIO: (n_ahead, 5) RATIO domain forecast exogenous (for recursive loop)
# denom_matrix : (n_ahead, 5) per-step 5-feature rolling-median denominators
# (col 0 recovers absolute AQI; cols 1-4 re-ratio-ify exogenous)
def get_latest_data_sequence(sequence_length, n_ahead, latitude, longitude):
required_columns = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co']
current_utc_time = datetime.now(pytz.utc)
window_start_time_ts = pd.Timestamp(
current_utc_time.replace(minute=0, second=0, microsecond=0) - timedelta(hours=sequence_length + 24 - 1)
)
window_end_time_ts = pd.Timestamp(current_utc_time.replace(minute=0, second=0, microsecond=0))
# --- Firebase (primary, if fully populated with >=336h) ---
fb_processed = get_firebase_data(sequence_length, n_ahead, latitude, longitude)
if fb_processed is not None:
print("Firebase history sufficient ΓÇö computing ratio sequence from Firebase.")
result = _build_ratio_sequence_from_processed(
fb_processed, sequence_length, n_ahead,
window_start_time_ts, window_end_time_ts, required_columns
)
if result[0] is not None:
return result
print("Firebase ratio build failed (gaps/NaN/thin window). Falling back to Open-Meteo.")
# --- Open-Meteo fallback (unchanged fetch, then shared helper) ---
print("Firebase data retrieval failed or unavailable. Falling back to OpenMeteo API...")
print(f"Attempting to retrieve data for the last {sequence_length} hours from Open-Meteo for Lat: {latitude}, Lon: {longitude}")
current_utc_time = datetime.now(pytz.utc)
print(f"Current UTC time on server for API calls: {current_utc_time.strftime('%Y-%m-%d %H:%M:%S UTC')}")
# Need enough history to compute rolling(336).median().shift(n_ahead) across ALL
# 24 input steps, plus generous margin so the ratio window isn't NaN.
api_fetch_past_hours = sequence_length + ROLLING_WINDOW + n_ahead + 72
processing_window_hours = sequence_length + 24
forecast_hours = min(n_ahead, MAX_N_AHEAD)
print(f"Requesting past {api_fetch_past_hours}h and forecast {forecast_hours}h from APIs.")
air_quality_url = "https://air-quality-api.open-meteo.com/v1/air-quality"
air_quality_params = {
"latitude": latitude,
"longitude": longitude,
"hourly": ["pm2_5", "pm10", "carbon_monoxide"],
"timezone": "auto",
"past_hours": api_fetch_past_hours,
"forecast_hours": forecast_hours,
}
weather_url = "https://api.open-meteo.com/v1/forecast"
weather_params = {
"latitude": latitude,
"longitude": longitude,
"hourly": ["temperature_2m"],
"timezone": "auto",
"past_hours": api_fetch_past_hours,
"forecast_hours": forecast_hours,
}
try:
print(f"Fetching air quality data from: {air_quality_url}")
air_quality_data = get_open_meteo_cached_json(air_quality_url, air_quality_params)
print("Air quality data retrieved.")
print(f"Fetching temperature data from: {weather_url}")
weather_data = get_open_meteo_cached_json(weather_url, weather_params)
print("Temperature data retrieved.")
if 'hourly' not in air_quality_data or 'time' not in air_quality_data['hourly']:
print("Error: 'hourly'/'time' missing in air quality response.")
return None, None, None, None
df_aq = pd.DataFrame(air_quality_data['hourly'])
if 'time' not in df_aq.columns and not df_aq.empty:
return None, None, None, None
if not df_aq.empty:
df_aq['time'] = pd.to_datetime(df_aq['time'])
df_aq.set_index('time', inplace=True)
print(f"Processed df_aq. Shape: {df_aq.shape}.")
if 'hourly' not in weather_data or 'time' not in weather_data['hourly']:
print("Error: 'hourly'/'time' missing in weather response.")
return None, None, None, None
df_temp = pd.DataFrame(weather_data['hourly'])
if 'time' not in df_temp.columns and not df_temp.empty:
return None, None, None, None
if not df_temp.empty:
df_temp['time'] = pd.to_datetime(df_temp['time'])
df_temp.set_index('time', inplace=True)
print(f"Processed df_temp. Shape: {df_temp.shape}.")
if df_aq.empty or df_temp.empty:
print("Error: One or both dataframes empty before merge.")
return None, None, None, None
df_merged = df_aq.merge(df_temp, left_index=True, right_index=True, how='inner')
print(f"DataFrames merged. Shape: {df_merged.shape}")
if df_merged.empty:
print("Error: Inner merge produced empty DataFrame.")
return None, None, None, None
df_processed = df_merged.resample('h').mean()
df_processed.index = pd.DatetimeIndex(df_processed.index)
df_processed = df_processed.ffill().bfill()
df_processed.rename(columns={'pm2_5': 'pm25', 'carbon_monoxide': 'co', 'temperature_2m': 'temp'}, inplace=True)
for col in ['pm25', 'pm10', 'co']:
if col not in df_processed.columns:
print(f"Warning: '{col}' missing after rename. Adding as NaN.")
df_processed[col] = np.nan
df_processed['calculated_aqi'] = df_processed.apply(lambda row: calculate_overall_aqi(row, aqi_breakpoints), axis=1)
required_columns = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co']
for col in required_columns:
if col not in df_processed.columns:
df_processed[col] = np.nan
df_processed = df_processed[required_columns].copy()
# Use the UTC offset from Open-Meteo to localize timestamps to local time,
# so slicing and rolling-median alignment happen on correct local-hour boundaries.
utc_offset_seconds = air_quality_data.get('utc_offset_seconds', 0)
local_tz = pytz.FixedOffset(utc_offset_seconds // 60)
if df_processed.index.tz is None:
df_processed.index = df_processed.index.tz_localize(local_tz)
else:
df_processed.index = df_processed.index.tz_convert(local_tz)
return _build_ratio_sequence_from_processed(
df_processed, sequence_length, n_ahead,
window_start_time_ts, window_end_time_ts, required_columns
)
except requests.exceptions.RequestException as e:
print(f"API Request Error: {e}")
traceback.print_exc()
return None, None, None, None
except Exception as e:
print(f"Unexpected error during data retrieval: {e}")
traceback.print_exc()
return None, None, None, None
# --- Paths ---
MODEL_PATH = 'best_model_TKAN_nahead_1.keras'
INPUT_SCALER_ATTR_PATH = 'input_scaler_attributes.json'
TARGET_SCALER_ATTR_PATH = 'target_scaler_attributes.json'
Y_SCALER_TRAIN_PATH = 'y_scaler_train.npy'
# --- Load scalers and model ---
input_scaler = None
target_scaler = None
model = None
try:
with open(INPUT_SCALER_ATTR_PATH, 'r') as f:
input_attrs = json.load(f)
input_scaler = MinMaxScaler()
input_scaler.load_attributes(input_attrs)
print("Input scaler loaded manually.")
with open(TARGET_SCALER_ATTR_PATH, 'r') as f:
target_attrs = json.load(f)
target_scaler = MinMaxScaler()
target_scaler.load_attributes(target_attrs)
print("Target scaler loaded manually.")
y_scaler_train = np.load(Y_SCALER_TRAIN_PATH)
print("y_scaler_train numpy array loaded.")
except FileNotFoundError as e:
print(f"Error loading scaler files (FileNotFoundError): {e}")
except Exception as e:
print(f"Error during scaler loading: {e}")
traceback.print_exc()
custom_objects = {"TKAN": TKANLayer, "tkan>TKAN": TKANLayer}
try:
print(f"Loading model from {MODEL_PATH}...")
with custom_object_scope(custom_objects):
model = load_model(MODEL_PATH, compile=False, custom_objects=custom_objects)
print("Model loaded successfully.")
except FileNotFoundError:
print(f"Error: Model file not found at {MODEL_PATH}.")
except ValueError as e:
print(f"Error loading model (ValueError): {e}")
traceback.print_exc()
except Exception as e:
print(f"Unexpected error during model loading: {e}")
traceback.print_exc()
# --- Startup contract diagnostic ---
if model is not None:
print(f"[CONTRACT] model.input_shape = {model.input_shape}")
print(f"[CONTRACT] model.output_shape = {model.output_shape}")
if input_scaler is not None and input_scaler.min_ is not None:
print(f"[CONTRACT] input_scaler.min_ = {np.asarray(input_scaler.min_).ravel()}")
print(f"[CONTRACT] input_scaler.max_ = {np.asarray(input_scaler.max_).ravel()}")
if target_scaler is not None and target_scaler.min_ is not None:
print(f"[CONTRACT] target_scaler.min_ = {np.asarray(target_scaler.min_).ravel()}")
print(f"[CONTRACT] target_scaler.max_ = {np.asarray(target_scaler.max_).ravel()}")
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
class PredictionRequest(BaseModel):
latitude: float
longitude: float
pm25: Optional[float] = None
pm10: Optional[float] = None
co: Optional[float] = None
temp: Optional[float] = None
n_ahead: int = Field(default=24, ge=1, le=MAX_N_AHEAD)
class Prediction(BaseModel):
timestamp: str
aqi: float
class PredictionResponse(BaseModel):
status: str
message: str
predictions: Optional[List[Prediction]] = None
@app.post("/predict", response_model=PredictionResponse)
async def predict_aqi_endpoint(request: PredictionRequest):
if model is None or input_scaler is None or target_scaler is None:
raise HTTPException(status_code=500, detail="Model or scalers not loaded. Check server logs.")
if model.input_shape is None or len(model.input_shape) < 3:
raise HTTPException(status_code=500, detail=f"Model has unexpected input shape: {model.input_shape}")
SEQUENCE_LENGTH = model.input_shape[1]
NUM_FEATURES = model.input_shape[2]
pipeline_features = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co']
if NUM_FEATURES != len(pipeline_features):
raise HTTPException(
status_code=500,
detail=(f"Feature contract mismatch: model expects {NUM_FEATURES} features, "
f"pipeline produces {len(pipeline_features)} ({pipeline_features}).")
)
# NOTE: current_sequence returned here is already in RATIO domain (model input space).
current_sequence_ratio, timestamps, future_exogenous_ratio, denom_matrix = get_latest_data_sequence(
SEQUENCE_LENGTH, request.n_ahead, request.latitude, request.longitude
)
if (current_sequence_ratio is None or timestamps is None
or future_exogenous_ratio is None or denom_matrix is None):
raise HTTPException(status_code=503, detail="Unable to retrieve data for prediction. Check server logs.")
prediction_timestamps = []
if timestamps and len(timestamps) > 0:
last_timestamp_of_sequence = timestamps[-1]
for i in range(request.n_ahead):
prediction_timestamps.append(last_timestamp_of_sequence + timedelta(hours=i + 1))
else:
now_utc = datetime.now(pytz.utc)
for i in range(request.n_ahead):
prediction_timestamps.append(now_utc + timedelta(hours=i + 1))
# --- USER-INPUT OVERRIDE DISABLED (was writing ABSOLUTE values into RATIO slots).
# The model input sequence is now in ratio domain. To re-enable, each user value
# must be divided by that timestep's rolling-median denominator (denom_matrix / the
# input-window denominator) before being written. Left off until verified. ---
# if request.pm25 is not None and ...:
# ... (compute current_aqi, ratio-transform, write into current_sequence_ratio[0,-1,:])
try:
predictions_list = []
current_sequence = current_sequence_ratio.copy() # RATIO domain
# Probe whether the model natively predicts multiple steps.
X_scaled_initial = input_scaler.transform(current_sequence)
scaled_prediction_initial = model.predict(X_scaled_initial, verbose=0)
actual_n_ahead = scaled_prediction_initial.shape[1] if len(scaled_prediction_initial.shape) > 1 else 1
if actual_n_ahead >= request.n_ahead:
# Model output is a SCALED RATIO. Recover absolute AQI:
# inverse-transform -> ratio, then multiply by per-step AQI denominator (col 0).
ratio = target_scaler.inverse_transform(
scaled_prediction_initial.reshape(1, actual_n_ahead, 1)
).flatten()
aqi_denom = denom_matrix[:request.n_ahead, 0]
predicted_aqi_values = ratio[:request.n_ahead] * aqi_denom
for i in range(request.n_ahead):
timestamp_str = prediction_timestamps[i].astimezone(pytz.utc).isoformat()
predictions_list.append(Prediction(timestamp=timestamp_str, aqi=float(predicted_aqi_values[i])))
else:
# Recursive single-step loop ΓÇö everything stays in RATIO domain.
for step in range(request.n_ahead):
X_scaled = input_scaler.transform(current_sequence)
scaled_prediction = model.predict(X_scaled, verbose=0)
# Scaled ratio -> ratio; absolute AQI = ratio * AQI denominator (col 0).
step_ratio = target_scaler.inverse_transform(
scaled_prediction.reshape(1, 1, 1)
).flatten()[0]
step_denom_aqi = denom_matrix[step, 0]
step_predicted_aqi = step_ratio * step_denom_aqi
timestamp_str = prediction_timestamps[step].astimezone(pytz.utc).isoformat()
predictions_list.append(Prediction(timestamp=timestamp_str, aqi=float(step_predicted_aqi)))
if step < request.n_ahead - 1:
# Slide window forward, staying in RATIO domain.
next_row = current_sequence[0, -1, :].copy()
# Exogenous features (cols 1-4) come from forecast, already ratio-domain.
next_row[1:] = future_exogenous_ratio[step, 1:]
# AQI feature (col 0): feed our predicted RATIO forward (step_ratio),
# NOT the absolute AQI ΓÇö the input sequence is ratio-domain.
next_row[0] = step_ratio
current_sequence[0, :-1, :] = current_sequence[0, 1:, :]
current_sequence[0, -1, :] = next_row
except Exception as e:
print(f"Error during prediction: {e}")
traceback.print_exc()
raise HTTPException(status_code=500, detail="Error during model prediction or inverse transform.")
return PredictionResponse(
status="success",
message=f"Prediction successful. Generated {len(predictions_list)} forecast values.",
predictions=predictions_list
)
@app.get("/")
async def read_root():
return {"message": "AQI Prediction API is running."}