Spaces:
Sleeping
Sleeping
Download app.py from vayu2601/project: direct link, hf CLI and curl.
- Browser
- Download file 33 kB
-
https://huggingface.co/spaces/vayu2601/project/resolve/main/app.py
- Command line
-
hf download hf://spaces/vayu2601/project/app.py
-
curl -L -o app.py https://huggingface.co/spaces/vayu2601/project/resolve/main/app.py
33 kB
| import os | |
| from typing import List, Optional, Tuple | |
| # Set environment variables for TensorFlow/Keras backend | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "-1" | |
| BACKEND = 'tensorflow' | |
| os.environ['KERAS_BACKEND'] = BACKEND | |
| from fastapi import FastAPI, HTTPException | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from pydantic import BaseModel, Field | |
| import numpy as np | |
| import tensorflow as tf | |
| from tensorflow.keras.models import load_model | |
| from tensorflow.keras.utils import custom_object_scope | |
| import requests | |
| import pandas as pd | |
| from datetime import datetime, timedelta | |
| import pytz | |
| import json | |
| import traceback | |
| import tempfile | |
| import time | |
| from threading import Lock | |
| import firebase_admin | |
| from firebase_admin import credentials, db | |
| # --- FIX 1: Fail loud on missing TKAN. No zero-returning stub. --- | |
| TKAN_IMPORT_OK = False | |
| TKAN_IMPORT_ERROR = None | |
| TKANLayer = None | |
| try: | |
| from tkan import TKAN as TKANLayer | |
| TKAN_IMPORT_OK = True | |
| print("TKAN imported from package 'tkan'.") | |
| except ImportError: | |
| try: | |
| from keras_efficient_kan import TKANLayer | |
| TKAN_IMPORT_OK = True | |
| print("TKAN imported from package 'keras_efficient_kan'.") | |
| except ImportError as e: | |
| TKAN_IMPORT_ERROR = e | |
| print("=" * 70) | |
| print("CRITICAL: Real TKAN layer could not be imported.") | |
| print("Neither 'tkan' nor 'keras_efficient_kan' is installed.") | |
| print("Refusing to serve until the correct package is installed.") | |
| print(f"Underlying import error: {e}") | |
| print("=" * 70) | |
| # Hard fail: never boot with a missing TKAN layer (would otherwise serve zeros). | |
| if not TKAN_IMPORT_OK: | |
| raise ImportError( | |
| "Real TKAN layer unavailable ΓÇö aborting startup to avoid serving " | |
| f"zero predictions. Install 'tkan' or 'keras_efficient_kan'. " | |
| f"Original error: {TKAN_IMPORT_ERROR}" | |
| ) | |
| # Constants | |
| MAX_N_AHEAD = 168 # Max 7 days of predictions | |
| SEQUENCE_LENGTH_MIN = 24 # Minimum historical data required | |
| ROLLING_WINDOW = 24 * 14 # 336h rolling-median window (CONFIRMED from training notebook) | |
| # Initialize Firebase using environment variables | |
| import tempfile | |
| firebase_credential_json = os.environ.get("FIREBASE_CREDENTIAL_JSON") | |
| firebase_initialized = False | |
| if firebase_credential_json: | |
| try: | |
| temp_dir = tempfile.gettempdir() | |
| firebase_credential_path = os.path.join(temp_dir, "firebase-credentials.json") | |
| with open(firebase_credential_path, "w", encoding="utf-8") as f: | |
| f.write(firebase_credential_json) | |
| print(f"Wrote Firebase credentials to writable temp path: {firebase_credential_path}") | |
| FIREBASE_DATABASE_URL = os.getenv("FIREBASE_DATABASE_URL") | |
| if not os.path.exists(firebase_credential_path): | |
| raise FileNotFoundError(f"Firebase credentials file not found at {firebase_credential_path}") | |
| cred = credentials.Certificate(firebase_credential_path) | |
| firebase_admin.initialize_app(cred, {'databaseURL': FIREBASE_DATABASE_URL}) | |
| firebase_initialized = True | |
| print("Firebase initialized successfully") | |
| except Exception as e: | |
| print(f"Firebase initialization failed: {e}") | |
| firebase_initialized = False | |
| else: | |
| print("Firebase credential not found in environment variables. Will use fallback (Open-Meteo only).") | |
| OPEN_METEO_CACHE_TTL_SECONDS = int(os.getenv("OPEN_METEO_CACHE_TTL_SECONDS", "900")) | |
| OPEN_METEO_CACHE_MAX_ENTRIES = int(os.getenv("OPEN_METEO_CACHE_MAX_ENTRIES", "64")) | |
| open_meteo_cache = {} | |
| open_meteo_cache_lock = Lock() | |
| def _normalize_params(params): | |
| normalized = [] | |
| for key, value in params.items(): | |
| if isinstance(value, list): | |
| normalized.append((key, tuple(value))) | |
| elif isinstance(value, dict): | |
| normalized.append((key, tuple(sorted(value.items())))) | |
| else: | |
| normalized.append((key, value)) | |
| return tuple(sorted(normalized)) | |
| def _make_open_meteo_cache_key(url, params): | |
| return (url, _normalize_params(params)) | |
| def _fetch_json_with_retry(url, params, max_attempts=3, backoff_factor=1.0): | |
| attempt = 1 | |
| while True: | |
| response = requests.get(url, params=params, timeout=30) | |
| if response.status_code == 429 or 500 <= response.status_code < 600: | |
| if attempt >= max_attempts: | |
| response.raise_for_status() | |
| sleep_seconds = backoff_factor * (2 ** (attempt - 1)) | |
| print(f"Open-Meteo request returned {response.status_code}. Retrying in {sleep_seconds:.1f}s (attempt {attempt}/{max_attempts})") | |
| time.sleep(sleep_seconds) | |
| attempt += 1 | |
| continue | |
| response.raise_for_status() | |
| return response.json() | |
| def get_open_meteo_cached_json(url, params): | |
| key = _make_open_meteo_cache_key(url, params) | |
| now = datetime.now(pytz.utc) | |
| with open_meteo_cache_lock: | |
| cached = open_meteo_cache.get(key) | |
| if cached and (now - cached["ts"]).total_seconds() < OPEN_METEO_CACHE_TTL_SECONDS: | |
| print(f"Open-Meteo cache hit for {url}") | |
| return cached["data"] | |
| data = _fetch_json_with_retry(url, params) | |
| with open_meteo_cache_lock: | |
| if len(open_meteo_cache) >= OPEN_METEO_CACHE_MAX_ENTRIES: | |
| oldest_key = min(open_meteo_cache.items(), key=lambda pair: pair[1]["ts"])[0] | |
| del open_meteo_cache[oldest_key] | |
| open_meteo_cache[key] = {"ts": now, "data": data} | |
| print(f"Open-Meteo cache set for {url}") | |
| return data | |
| class MinMaxScaler: | |
| def __init__(self, feature_axis=None, minmax_range=(0, 1)): | |
| self.feature_axis = feature_axis | |
| self.min_ = None | |
| self.max_ = None | |
| self.scale_ = None | |
| self.minmax_range = minmax_range | |
| def load_attributes(self, attributes): | |
| self.min_ = np.array(attributes['min_']) if isinstance(attributes['min_'], list) else attributes['min_'] | |
| self.max_ = np.array(attributes['max_']) if isinstance(attributes['max_'], list) else attributes['max_'] | |
| self.scale_ = np.array(attributes['scale_']) if isinstance(attributes['scale_'], list) else attributes['scale_'] | |
| self.minmax_range = tuple(attributes['minmax_range']) if isinstance(attributes['minmax_range'], list) else attributes['minmax_range'] | |
| # --- FIX 2: guard against zero scale (constant feature -> inf/nan). --- | |
| self.scale_ = np.where(np.asarray(self.scale_) == 0, 1, self.scale_) | |
| def fit(self, X): | |
| if X.ndim == 3 and self.feature_axis is not None: | |
| axis = tuple(i for i in range(X.ndim) if i != self.feature_axis) | |
| self.min_ = np.min(X, axis=axis) | |
| self.max_ = np.max(X, axis=axis) | |
| elif X.ndim == 2: | |
| self.min_ = np.min(X, axis=0) | |
| self.max_ = np.max(X, axis=0) | |
| elif X.ndim == 1: | |
| self.min_ = np.min(X) | |
| self.max_ = np.max(X) | |
| else: | |
| raise ValueError("Data must be 1D, 2D, or 3D.") | |
| self.scale_ = self.max_ - self.min_ | |
| self.scale_ = np.where(np.asarray(self.scale_) == 0, 1, self.scale_) | |
| return self | |
| def transform(self, X): | |
| if self.min_ is None or self.max_ is None or self.scale_ is None: | |
| raise ValueError("Scaler attributes not loaded or scaler not fitted.") | |
| X_scaled = (X - self.min_) / self.scale_ | |
| X_scaled = X_scaled * (self.minmax_range[1] - self.minmax_range[0]) + self.minmax_range[0] | |
| return X_scaled | |
| def inverse_transform(self, X_scaled): | |
| if self.min_ is None or self.max_ is None or self.scale_ is None: | |
| raise ValueError("Scaler attributes not loaded or scaler not fitted.") | |
| X = (X_scaled - self.minmax_range[0]) / (self.minmax_range[1] - self.minmax_range[0]) | |
| X = X * self.scale_ + self.min_ | |
| return X | |
| # --- AQI breakpoints and calculation functions --- | |
| # NOTE (confirmed caveat): CO values in the training data range 106-1869, far above | |
| # the top CO breakpoint of 17.0, so CO saturates. This unit mismatch is baked into | |
| # training; serving computes AQI identically so it stays self-consistent, but the | |
| # predicted "AQI" is NOT standards-compliant. Left unchanged deliberately. | |
| aqi_breakpoints = { | |
| 'pm25': [(0, 50, 0, 50), (51, 100, 51, 100), (101, 200, 101, 200), (201, 300, 201, 300)], | |
| 'pm10': [(0, 50, 0, 50), (51, 100, 51, 100), (101, 250, 101, 200), (251, 350, 201, 300)], | |
| 'co': [(0, 1.0, 0, 50), (1.1, 2.0, 51, 100), (2.1, 10.0, 101, 200), (10.1, 17.0, 201, 300)] | |
| } | |
| def calculate_sub_aqi(concentration, breakpoints): | |
| for i_low, i_high, c_low, c_high in breakpoints: | |
| if c_low <= concentration <= c_high: | |
| if c_high == c_low: | |
| return i_low | |
| return ((i_high - i_low) / (c_high - c_low)) * (concentration - c_low) + i_low | |
| if concentration < breakpoints[0][2]: | |
| return breakpoints[0][0] | |
| elif concentration > breakpoints[-1][3]: | |
| return breakpoints[-1][1] | |
| else: | |
| return np.nan | |
| def calculate_overall_aqi(row, aqi_breakpoints): | |
| sub_aqis = [] | |
| pollutant_mapping = { | |
| 'pm25': 'pm25', | |
| 'pm10': 'pm10', | |
| 'co': 'co', | |
| 'pm2_5': 'pm25', | |
| 'carbon_monoxide': 'co', | |
| } | |
| for api_pollutant, internal_pollutant in pollutant_mapping.items(): | |
| if api_pollutant in row: | |
| concentration = row[api_pollutant] | |
| if not pd.isna(concentration): | |
| sub_aqi = calculate_sub_aqi(concentration, aqi_breakpoints.get(internal_pollutant, [])) | |
| sub_aqis.append(sub_aqi) | |
| else: | |
| sub_aqis.append(np.nan) | |
| else: | |
| sub_aqis.append(np.nan) | |
| if sub_aqis and not all(pd.isna(sub_aqis)): | |
| return np.nanmax(sub_aqis) | |
| else: | |
| return np.nan | |
| def _build_ratio_sequence_from_processed(df_processed, sequence_length, n_ahead, | |
| window_start_time_ts, window_end_time_ts, | |
| required_columns): | |
| """ | |
| Shared ratio-transform pipeline used by BOTH Firebase and Open-Meteo paths. | |
| df_processed: hourly, absolute-valued, columns == required_columns, tz-aware UTC index. | |
| Returns (ratio_seq (1,seq,5), timestamps, future_exogenous_ratio (n_ahead,5), | |
| denom_matrix (n_ahead,5)) or (None, None, None, None) on failure. | |
| Mirrors training generate_data(): scaler_df = df.shift(n_ahead).rolling(336).median(); | |
| tmp_df = df / scaler_df. | |
| """ | |
| scaler_df_full = df_processed[required_columns].shift(n_ahead).rolling(ROLLING_WINDOW).median() | |
| ratio_df_full = df_processed[required_columns] / scaler_df_full | |
| rolling_median_aqi = scaler_df_full['calculated_aqi'] | |
| finite_median = rolling_median_aqi.dropna() | |
| print(f"Rolling-median series computed. Finite points: {len(finite_median)}") | |
| finite_scaler_rows = scaler_df_full.dropna() | |
| if len(finite_scaler_rows): | |
| fallback_denom_row = finite_scaler_rows.iloc[-1].values.astype(np.float64) | |
| else: | |
| fallback_denom_row = np.ones((len(required_columns),), dtype=np.float64) | |
| fallback_denom_row = np.where(fallback_denom_row != 0, fallback_denom_row, 1.0) | |
| # Recent absolute window (timestamps + last-row anchor + real forecast exogenous). | |
| abs_recent_all = df_processed[ | |
| (df_processed.index >= window_start_time_ts) & (df_processed.index <= window_end_time_ts) | |
| ].copy() | |
| abs_recent_all.dropna(inplace=True) | |
| print(f"Shape after dropna on recent window: {abs_recent_all.shape}") | |
| if len(abs_recent_all) < sequence_length: | |
| print(f"Error: Only {len(abs_recent_all)} valid points, need {sequence_length}.") | |
| return None, None, None, None | |
| abs_recent_df = abs_recent_all.tail(sequence_length).copy() | |
| # Ratio window = model input. NaN guard: 336h median must be finite across all steps. | |
| ratio_recent = ratio_df_full.reindex(abs_recent_df.index) | |
| if ratio_recent.isna().any().any(): | |
| print("Error: ratio window contains NaN ΓÇö 336h rolling median not finite across " | |
| "all input steps. Not enough clean history.") | |
| return None, None, None, None | |
| latest_ratio_df = ratio_recent.tail(sequence_length).copy() | |
| ratio_seq = latest_ratio_df.values.reshape(1, sequence_length, len(required_columns)) | |
| timestamps = latest_ratio_df.index.tolist() | |
| last_ts = abs_recent_df.index[-1] | |
| df_future = df_processed[df_processed.index > window_end_time_ts].copy() | |
| df_future.dropna(inplace=True) | |
| num_features = len(required_columns) | |
| future_exogenous_ratio = np.zeros((n_ahead, num_features), dtype=np.float64) | |
| denom_matrix = np.zeros((n_ahead, num_features), dtype=np.float64) | |
| last_known_abs = abs_recent_df.iloc[-1].values.astype(np.float64) | |
| for i in range(n_ahead): | |
| pred_ts = last_ts + pd.Timedelta(hours=i + 1) | |
| if pred_ts in scaler_df_full.index and not scaler_df_full.loc[pred_ts].isna().any(): | |
| denom_row = scaler_df_full.loc[pred_ts].values.astype(np.float64) | |
| else: | |
| denom_row = fallback_denom_row.copy() | |
| denom_row = np.where(denom_row != 0, denom_row, 1.0) | |
| denom_matrix[i] = denom_row | |
| exo_abs = df_future.iloc[i].values.astype(np.float64) if i < len(df_future) else last_known_abs | |
| future_exogenous_ratio[i] = exo_abs / denom_row | |
| return ratio_seq, timestamps, future_exogenous_ratio, denom_matrix | |
| # --- Function to retrieve data from Firebase --- | |
| def get_firebase_data(sequence_length: int, n_ahead: int, latitude: float, longitude: float): | |
| """ | |
| Fetch enough Firebase history to compute the 336h rolling median, resample hourly, | |
| compute calculated_aqi, and return an absolute-valued df_processed (or None). | |
| Schema assumed from prior code: key 'datetime' as '%Y-%m-%dT%H:%M', pollutant keys | |
| pm2_5 / pm10 / carbon_monoxide / temp. Adjust here if your DB differs. | |
| """ | |
| if not firebase_initialized: | |
| return None | |
| try: | |
| print("Attempting to retrieve data from Firebase RTDB...") | |
| ref = db.reference('/AQIData') | |
| current_utc_time = datetime.now(pytz.utc) | |
| # Need 336h + input window + margin, same as the Open-Meteo path. | |
| fetch_hours = sequence_length + ROLLING_WINDOW + n_ahead + 72 | |
| window_start = current_utc_time - timedelta(hours=fetch_hours) | |
| window_end = current_utc_time | |
| firebase_data = ref.order_by_child('datetime').start_at( | |
| window_start.strftime('%Y-%m-%dT%H:%M') | |
| ).end_at(window_end.strftime('%Y-%m-%dT%H:%M')).get() | |
| if not firebase_data: | |
| print("No data found in Firebase for the specified time range") | |
| return None | |
| data_list = [] | |
| for key, sensor_data in firebase_data.items(): | |
| try: | |
| datetime_str = sensor_data.get('datetime') | |
| if isinstance(datetime_str, str): | |
| ts = pytz.utc.localize(datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M')) | |
| data_list.append({ | |
| 'time': ts, | |
| 'pm25': sensor_data.get('pm2_5', np.nan), | |
| 'pm10': sensor_data.get('pm10', np.nan), | |
| 'co': sensor_data.get('carbon_monoxide', np.nan), | |
| 'temp': sensor_data.get('temp', np.nan), | |
| }) | |
| else: | |
| print(f"Warning: item {key} missing/invalid 'datetime': {datetime_str}") | |
| except ValueError as ve: | |
| print(f"Error parsing datetime '{datetime_str}' for key {key}: {ve}") | |
| continue | |
| except Exception as e: | |
| print(f"Unexpected error processing item {key}: {e}") | |
| continue | |
| if not data_list: | |
| print("No valid data points parsed from Firebase.") | |
| return None | |
| df = pd.DataFrame(data_list).set_index('time').sort_index() | |
| print(f"Retrieved {len(df)} data points from Firebase") | |
| df_hourly = df.resample('h').mean() | |
| # Gap check kept from your original logic. | |
| time_diffs = df_hourly.index.to_series().diff() | |
| max_gap = time_diffs.max() | |
| if pd.notna(max_gap) and max_gap > timedelta(hours=1, minutes=30): | |
| print(f"Data has gaps larger than 1.5 hours. Max gap: {max_gap}") | |
| return None | |
| df_hourly = df_hourly.ffill().bfill() | |
| df_hourly['calculated_aqi'] = df_hourly.apply( | |
| lambda row: calculate_overall_aqi(row, aqi_breakpoints), axis=1 | |
| ) | |
| required_columns = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co'] | |
| for col in required_columns: | |
| if col not in df_hourly.columns: | |
| df_hourly[col] = np.nan | |
| df_processed = df_hourly[required_columns].copy() | |
| if df_processed.index.tz is None: | |
| df_processed.index = df_processed.index.tz_localize('UTC') | |
| # Need at least 336h + window to have any finite median at all. | |
| if len(df_processed) < ROLLING_WINDOW + sequence_length: | |
| print(f"Firebase returned {len(df_processed)} hourly points; need " | |
| f"{ROLLING_WINDOW + sequence_length} for a valid rolling median. " | |
| f"Falling back to Open-Meteo.") | |
| return None | |
| print(f"Firebase df_processed ready. Shape: {df_processed.shape}") | |
| return df_processed | |
| except Exception as e: | |
| print(f"Error retrieving data from Firebase: {e}") | |
| traceback.print_exc() | |
| return None | |
| # --- Data retrieval function --- | |
| # Returns: (ratio_input_sequence, timestamps, future_exogenous_RATIO, denom_matrix) | |
| # ratio_input_sequence : (1, seq_len, 5) RATIO domain ΓÇö what the model consumes | |
| # future_exogenous_RATIO: (n_ahead, 5) RATIO domain forecast exogenous (for recursive loop) | |
| # denom_matrix : (n_ahead, 5) per-step 5-feature rolling-median denominators | |
| # (col 0 recovers absolute AQI; cols 1-4 re-ratio-ify exogenous) | |
| def get_latest_data_sequence(sequence_length, n_ahead, latitude, longitude): | |
| required_columns = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co'] | |
| current_utc_time = datetime.now(pytz.utc) | |
| window_start_time_ts = pd.Timestamp( | |
| current_utc_time.replace(minute=0, second=0, microsecond=0) - timedelta(hours=sequence_length + 24 - 1) | |
| ) | |
| window_end_time_ts = pd.Timestamp(current_utc_time.replace(minute=0, second=0, microsecond=0)) | |
| # --- Firebase (primary, if fully populated with >=336h) --- | |
| fb_processed = get_firebase_data(sequence_length, n_ahead, latitude, longitude) | |
| if fb_processed is not None: | |
| print("Firebase history sufficient ΓÇö computing ratio sequence from Firebase.") | |
| result = _build_ratio_sequence_from_processed( | |
| fb_processed, sequence_length, n_ahead, | |
| window_start_time_ts, window_end_time_ts, required_columns | |
| ) | |
| if result[0] is not None: | |
| return result | |
| print("Firebase ratio build failed (gaps/NaN/thin window). Falling back to Open-Meteo.") | |
| # --- Open-Meteo fallback (unchanged fetch, then shared helper) --- | |
| print("Firebase data retrieval failed or unavailable. Falling back to OpenMeteo API...") | |
| print(f"Attempting to retrieve data for the last {sequence_length} hours from Open-Meteo for Lat: {latitude}, Lon: {longitude}") | |
| current_utc_time = datetime.now(pytz.utc) | |
| print(f"Current UTC time on server for API calls: {current_utc_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") | |
| # Need enough history to compute rolling(336).median().shift(n_ahead) across ALL | |
| # 24 input steps, plus generous margin so the ratio window isn't NaN. | |
| api_fetch_past_hours = sequence_length + ROLLING_WINDOW + n_ahead + 72 | |
| processing_window_hours = sequence_length + 24 | |
| forecast_hours = min(n_ahead, MAX_N_AHEAD) | |
| print(f"Requesting past {api_fetch_past_hours}h and forecast {forecast_hours}h from APIs.") | |
| air_quality_url = "https://air-quality-api.open-meteo.com/v1/air-quality" | |
| air_quality_params = { | |
| "latitude": latitude, | |
| "longitude": longitude, | |
| "hourly": ["pm2_5", "pm10", "carbon_monoxide"], | |
| "timezone": "auto", | |
| "past_hours": api_fetch_past_hours, | |
| "forecast_hours": forecast_hours, | |
| } | |
| weather_url = "https://api.open-meteo.com/v1/forecast" | |
| weather_params = { | |
| "latitude": latitude, | |
| "longitude": longitude, | |
| "hourly": ["temperature_2m"], | |
| "timezone": "auto", | |
| "past_hours": api_fetch_past_hours, | |
| "forecast_hours": forecast_hours, | |
| } | |
| try: | |
| print(f"Fetching air quality data from: {air_quality_url}") | |
| air_quality_data = get_open_meteo_cached_json(air_quality_url, air_quality_params) | |
| print("Air quality data retrieved.") | |
| print(f"Fetching temperature data from: {weather_url}") | |
| weather_data = get_open_meteo_cached_json(weather_url, weather_params) | |
| print("Temperature data retrieved.") | |
| if 'hourly' not in air_quality_data or 'time' not in air_quality_data['hourly']: | |
| print("Error: 'hourly'/'time' missing in air quality response.") | |
| return None, None, None, None | |
| df_aq = pd.DataFrame(air_quality_data['hourly']) | |
| if 'time' not in df_aq.columns and not df_aq.empty: | |
| return None, None, None, None | |
| if not df_aq.empty: | |
| df_aq['time'] = pd.to_datetime(df_aq['time']) | |
| df_aq.set_index('time', inplace=True) | |
| print(f"Processed df_aq. Shape: {df_aq.shape}.") | |
| if 'hourly' not in weather_data or 'time' not in weather_data['hourly']: | |
| print("Error: 'hourly'/'time' missing in weather response.") | |
| return None, None, None, None | |
| df_temp = pd.DataFrame(weather_data['hourly']) | |
| if 'time' not in df_temp.columns and not df_temp.empty: | |
| return None, None, None, None | |
| if not df_temp.empty: | |
| df_temp['time'] = pd.to_datetime(df_temp['time']) | |
| df_temp.set_index('time', inplace=True) | |
| print(f"Processed df_temp. Shape: {df_temp.shape}.") | |
| if df_aq.empty or df_temp.empty: | |
| print("Error: One or both dataframes empty before merge.") | |
| return None, None, None, None | |
| df_merged = df_aq.merge(df_temp, left_index=True, right_index=True, how='inner') | |
| print(f"DataFrames merged. Shape: {df_merged.shape}") | |
| if df_merged.empty: | |
| print("Error: Inner merge produced empty DataFrame.") | |
| return None, None, None, None | |
| df_processed = df_merged.resample('h').mean() | |
| df_processed.index = pd.DatetimeIndex(df_processed.index) | |
| df_processed = df_processed.ffill().bfill() | |
| df_processed.rename(columns={'pm2_5': 'pm25', 'carbon_monoxide': 'co', 'temperature_2m': 'temp'}, inplace=True) | |
| for col in ['pm25', 'pm10', 'co']: | |
| if col not in df_processed.columns: | |
| print(f"Warning: '{col}' missing after rename. Adding as NaN.") | |
| df_processed[col] = np.nan | |
| df_processed['calculated_aqi'] = df_processed.apply(lambda row: calculate_overall_aqi(row, aqi_breakpoints), axis=1) | |
| required_columns = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co'] | |
| for col in required_columns: | |
| if col not in df_processed.columns: | |
| df_processed[col] = np.nan | |
| df_processed = df_processed[required_columns].copy() | |
| # Use the UTC offset from Open-Meteo to localize timestamps to local time, | |
| # so slicing and rolling-median alignment happen on correct local-hour boundaries. | |
| utc_offset_seconds = air_quality_data.get('utc_offset_seconds', 0) | |
| local_tz = pytz.FixedOffset(utc_offset_seconds // 60) | |
| if df_processed.index.tz is None: | |
| df_processed.index = df_processed.index.tz_localize(local_tz) | |
| else: | |
| df_processed.index = df_processed.index.tz_convert(local_tz) | |
| return _build_ratio_sequence_from_processed( | |
| df_processed, sequence_length, n_ahead, | |
| window_start_time_ts, window_end_time_ts, required_columns | |
| ) | |
| except requests.exceptions.RequestException as e: | |
| print(f"API Request Error: {e}") | |
| traceback.print_exc() | |
| return None, None, None, None | |
| except Exception as e: | |
| print(f"Unexpected error during data retrieval: {e}") | |
| traceback.print_exc() | |
| return None, None, None, None | |
| # --- Paths --- | |
| MODEL_PATH = 'best_model_TKAN_nahead_1.keras' | |
| INPUT_SCALER_ATTR_PATH = 'input_scaler_attributes.json' | |
| TARGET_SCALER_ATTR_PATH = 'target_scaler_attributes.json' | |
| Y_SCALER_TRAIN_PATH = 'y_scaler_train.npy' | |
| # --- Load scalers and model --- | |
| input_scaler = None | |
| target_scaler = None | |
| model = None | |
| try: | |
| with open(INPUT_SCALER_ATTR_PATH, 'r') as f: | |
| input_attrs = json.load(f) | |
| input_scaler = MinMaxScaler() | |
| input_scaler.load_attributes(input_attrs) | |
| print("Input scaler loaded manually.") | |
| with open(TARGET_SCALER_ATTR_PATH, 'r') as f: | |
| target_attrs = json.load(f) | |
| target_scaler = MinMaxScaler() | |
| target_scaler.load_attributes(target_attrs) | |
| print("Target scaler loaded manually.") | |
| y_scaler_train = np.load(Y_SCALER_TRAIN_PATH) | |
| print("y_scaler_train numpy array loaded.") | |
| except FileNotFoundError as e: | |
| print(f"Error loading scaler files (FileNotFoundError): {e}") | |
| except Exception as e: | |
| print(f"Error during scaler loading: {e}") | |
| traceback.print_exc() | |
| custom_objects = {"TKAN": TKANLayer, "tkan>TKAN": TKANLayer} | |
| try: | |
| print(f"Loading model from {MODEL_PATH}...") | |
| with custom_object_scope(custom_objects): | |
| model = load_model(MODEL_PATH, compile=False, custom_objects=custom_objects) | |
| print("Model loaded successfully.") | |
| except FileNotFoundError: | |
| print(f"Error: Model file not found at {MODEL_PATH}.") | |
| except ValueError as e: | |
| print(f"Error loading model (ValueError): {e}") | |
| traceback.print_exc() | |
| except Exception as e: | |
| print(f"Unexpected error during model loading: {e}") | |
| traceback.print_exc() | |
| # --- Startup contract diagnostic --- | |
| if model is not None: | |
| print(f"[CONTRACT] model.input_shape = {model.input_shape}") | |
| print(f"[CONTRACT] model.output_shape = {model.output_shape}") | |
| if input_scaler is not None and input_scaler.min_ is not None: | |
| print(f"[CONTRACT] input_scaler.min_ = {np.asarray(input_scaler.min_).ravel()}") | |
| print(f"[CONTRACT] input_scaler.max_ = {np.asarray(input_scaler.max_).ravel()}") | |
| if target_scaler is not None and target_scaler.min_ is not None: | |
| print(f"[CONTRACT] target_scaler.min_ = {np.asarray(target_scaler.min_).ravel()}") | |
| print(f"[CONTRACT] target_scaler.max_ = {np.asarray(target_scaler.max_).ravel()}") | |
| app = FastAPI() | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=["*"], | |
| allow_credentials=True, | |
| allow_methods=["*"], | |
| allow_headers=["*"], | |
| ) | |
| class PredictionRequest(BaseModel): | |
| latitude: float | |
| longitude: float | |
| pm25: Optional[float] = None | |
| pm10: Optional[float] = None | |
| co: Optional[float] = None | |
| temp: Optional[float] = None | |
| n_ahead: int = Field(default=24, ge=1, le=MAX_N_AHEAD) | |
| class Prediction(BaseModel): | |
| timestamp: str | |
| aqi: float | |
| class PredictionResponse(BaseModel): | |
| status: str | |
| message: str | |
| predictions: Optional[List[Prediction]] = None | |
| async def predict_aqi_endpoint(request: PredictionRequest): | |
| if model is None or input_scaler is None or target_scaler is None: | |
| raise HTTPException(status_code=500, detail="Model or scalers not loaded. Check server logs.") | |
| if model.input_shape is None or len(model.input_shape) < 3: | |
| raise HTTPException(status_code=500, detail=f"Model has unexpected input shape: {model.input_shape}") | |
| SEQUENCE_LENGTH = model.input_shape[1] | |
| NUM_FEATURES = model.input_shape[2] | |
| pipeline_features = ['calculated_aqi', 'temp', 'pm25', 'pm10', 'co'] | |
| if NUM_FEATURES != len(pipeline_features): | |
| raise HTTPException( | |
| status_code=500, | |
| detail=(f"Feature contract mismatch: model expects {NUM_FEATURES} features, " | |
| f"pipeline produces {len(pipeline_features)} ({pipeline_features}).") | |
| ) | |
| # NOTE: current_sequence returned here is already in RATIO domain (model input space). | |
| current_sequence_ratio, timestamps, future_exogenous_ratio, denom_matrix = get_latest_data_sequence( | |
| SEQUENCE_LENGTH, request.n_ahead, request.latitude, request.longitude | |
| ) | |
| if (current_sequence_ratio is None or timestamps is None | |
| or future_exogenous_ratio is None or denom_matrix is None): | |
| raise HTTPException(status_code=503, detail="Unable to retrieve data for prediction. Check server logs.") | |
| prediction_timestamps = [] | |
| if timestamps and len(timestamps) > 0: | |
| last_timestamp_of_sequence = timestamps[-1] | |
| for i in range(request.n_ahead): | |
| prediction_timestamps.append(last_timestamp_of_sequence + timedelta(hours=i + 1)) | |
| else: | |
| now_utc = datetime.now(pytz.utc) | |
| for i in range(request.n_ahead): | |
| prediction_timestamps.append(now_utc + timedelta(hours=i + 1)) | |
| # --- USER-INPUT OVERRIDE DISABLED (was writing ABSOLUTE values into RATIO slots). | |
| # The model input sequence is now in ratio domain. To re-enable, each user value | |
| # must be divided by that timestep's rolling-median denominator (denom_matrix / the | |
| # input-window denominator) before being written. Left off until verified. --- | |
| # if request.pm25 is not None and ...: | |
| # ... (compute current_aqi, ratio-transform, write into current_sequence_ratio[0,-1,:]) | |
| try: | |
| predictions_list = [] | |
| current_sequence = current_sequence_ratio.copy() # RATIO domain | |
| # Probe whether the model natively predicts multiple steps. | |
| X_scaled_initial = input_scaler.transform(current_sequence) | |
| scaled_prediction_initial = model.predict(X_scaled_initial, verbose=0) | |
| actual_n_ahead = scaled_prediction_initial.shape[1] if len(scaled_prediction_initial.shape) > 1 else 1 | |
| if actual_n_ahead >= request.n_ahead: | |
| # Model output is a SCALED RATIO. Recover absolute AQI: | |
| # inverse-transform -> ratio, then multiply by per-step AQI denominator (col 0). | |
| ratio = target_scaler.inverse_transform( | |
| scaled_prediction_initial.reshape(1, actual_n_ahead, 1) | |
| ).flatten() | |
| aqi_denom = denom_matrix[:request.n_ahead, 0] | |
| predicted_aqi_values = ratio[:request.n_ahead] * aqi_denom | |
| for i in range(request.n_ahead): | |
| timestamp_str = prediction_timestamps[i].astimezone(pytz.utc).isoformat() | |
| predictions_list.append(Prediction(timestamp=timestamp_str, aqi=float(predicted_aqi_values[i]))) | |
| else: | |
| # Recursive single-step loop ΓÇö everything stays in RATIO domain. | |
| for step in range(request.n_ahead): | |
| X_scaled = input_scaler.transform(current_sequence) | |
| scaled_prediction = model.predict(X_scaled, verbose=0) | |
| # Scaled ratio -> ratio; absolute AQI = ratio * AQI denominator (col 0). | |
| step_ratio = target_scaler.inverse_transform( | |
| scaled_prediction.reshape(1, 1, 1) | |
| ).flatten()[0] | |
| step_denom_aqi = denom_matrix[step, 0] | |
| step_predicted_aqi = step_ratio * step_denom_aqi | |
| timestamp_str = prediction_timestamps[step].astimezone(pytz.utc).isoformat() | |
| predictions_list.append(Prediction(timestamp=timestamp_str, aqi=float(step_predicted_aqi))) | |
| if step < request.n_ahead - 1: | |
| # Slide window forward, staying in RATIO domain. | |
| next_row = current_sequence[0, -1, :].copy() | |
| # Exogenous features (cols 1-4) come from forecast, already ratio-domain. | |
| next_row[1:] = future_exogenous_ratio[step, 1:] | |
| # AQI feature (col 0): feed our predicted RATIO forward (step_ratio), | |
| # NOT the absolute AQI ΓÇö the input sequence is ratio-domain. | |
| next_row[0] = step_ratio | |
| current_sequence[0, :-1, :] = current_sequence[0, 1:, :] | |
| current_sequence[0, -1, :] = next_row | |
| except Exception as e: | |
| print(f"Error during prediction: {e}") | |
| traceback.print_exc() | |
| raise HTTPException(status_code=500, detail="Error during model prediction or inverse transform.") | |
| return PredictionResponse( | |
| status="success", | |
| message=f"Prediction successful. Generated {len(predictions_list)} forecast values.", | |
| predictions=predictions_list | |
| ) | |
| async def read_root(): | |
| return {"message": "AQI Prediction API is running."} | |