File size: 12,544 Bytes
3d2a487
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
"""24-Hour Ahead Flood Forecast Inference β€” Genuine Future Forecasting.



Strategy

--------

1. Fetch a 48-row window from the Open-Meteo FORECAST API:

     past_hours=24   β†’  real observed data (grounding lag features)

     forecast_hours=24 β†’ NWP model output (GFS/IFS)

   The DataFrame includes an ``is_forecast`` boolean column.



2. Engineer rolling/lag features across the full 48 rows so that lag

   features at the START of the forecast window are computed from real data.



3. Load the nowcast model (models/best.pt, PR-AUC=0.9923) β€” this model

   sees a 24-row LSTM window and outputs "flood risk right now".



4. Slide that 24-row LSTM window across each of the 24 future rows:

     window ending at future step t  β†’  "flood risk at time t"



5. Return max(probabilities) β€” the answer to "any flood in the next 24h?"

   Also return the peak_time: when the highest risk is predicted.

"""
from __future__ import annotations

import functools
from typing import Any

import joblib
import numpy as np
import pandas as pd
import torch

from src.api.weather_fetcher import fetch_forecast_window
from src.config import load_default_config, PROJECT_ROOT
from src.pipeline.ingestion.loader import FEATURE_COLUMNS
from src.pipeline.training.model import build_model

NOWCAST_WINDOW    = 24    # must match best.pt training WINDOW=24
DEFAULT_THRESHOLD = 0.5   # fallback if not stored in checkpoint
PAST_HOURS        = 96    # 4 days of history so Rain_24h/Rain_12h features
                          # capture cumulative rainfall, not just last hour
FORECAST_HOURS    = 24    # future steps to evaluate


# ─────────────────────────────────────────────────────────────────────────────
# Alert thresholds
# ─────────────────────────────────────────────────────────────────────────────

def _alert_level(prob: float, threshold: float = 0.5) -> str:
    """Alert level anchored to the F1-optimal flood threshold.



    The model baseline for zero-signal (OOD) inputs is sigmoid(0)=0.5.

    Setting LOW boundary at threshold*0.70 (~0.509) means that uncertain

    non-events (0.500x) correctly show as LOW, not MODERATE.



      LOW      prob < threshold * 0.70   (~0.509)

      MODERATE threshold*0.70 to threshold  (0.509–0.728)

      HIGH     threshold to threshold*1.15  (0.728–0.837)

      CRITICAL prob >= threshold * 1.15     (>0.837)

    """
    if prob < threshold * 0.70:
        return "LOW"
    if prob < threshold:
        return "MODERATE"
    if prob < threshold * 1.15:
        return "HIGH"
    return "CRITICAL"


# ─────────────────────────────────────────────────────────────────────────────
# Assets β€” loaded once, cached
# ─────────────────────────────────────────────────────────────────────────────

@functools.lru_cache(maxsize=1)
def _load_assets() -> tuple[Any, Any, dict, float]:
    """Return (nowcast_model, scaler, cfg, threshold) β€” cached after first call."""
    cfg    = load_default_config()
    device = torch.device("cpu")

    # Use forecast_24h.pt β€” trained with Focal Loss + WeightedRandomSampler on
    # full 5.8M rows, stores calibrated optimal_threshold, PR-AUC=0.9597.
    # Both are horizon=0 nowcast models; the difference is training quality.
    ckpt_path = PROJECT_ROOT / cfg["paths"].get("models_dir", "models") / "forecast_24h.pt"
    if not ckpt_path.exists():
        raise RuntimeError(
            f"Forecast model not found at '{ckpt_path}'. "
            "Run: python -m src.pipeline.training.run_train_forecast"
        )

    model   = build_model(cfg, num_features=len(FEATURE_COLUMNS)).to(device)
    payload = torch.load(ckpt_path, map_location=device, weights_only=True)
    model.load_state_dict(payload["model_state_dict"])
    model.eval()
    threshold = float(payload.get("optimal_threshold", DEFAULT_THRESHOLD))
    print(
        f"[forecast] Loaded model from '{ckpt_path}'  "
        f"(epoch {payload.get('epoch', '?')}  threshold={threshold:.4f}  "
        f"pr_auc={payload.get('metrics', {}).get('test_pr_auc', '?')})"
    )

    scaler_path = PROJECT_ROOT / cfg["paths"]["scaler"]
    if not scaler_path.exists():
        raise RuntimeError(f"Scaler not found at '{scaler_path}'. Run 'dvc repro' first.")
    scaler = joblib.load(scaler_path)
    print(f"[forecast] Loaded scaler from '{scaler_path}'")

    return model, scaler, cfg, threshold


# ─────────────────────────────────────────────────────────────────────────────
# Feature engineering (shared with inference.py)
# ─────────────────────────────────────────────────────────────────────────────

def _engineer_features(df: pd.DataFrame) -> pd.DataFrame:
    """Compute rolling / lag features.  Preserves ``is_forecast`` if present."""
    df = df.copy().reset_index(drop=True)
    p  = df["Precipitation_mm"]
    sm = df["Soil_Moisture"]
    t  = df["Temperature_C"]

    df["Rain_3h"]       = p.rolling(window=3,  min_periods=1).sum()
    df["Rain_6h"]       = p.rolling(window=6,  min_periods=1).sum()
    df["Rain_12h"]      = p.rolling(window=12, min_periods=1).sum()
    df["Rain_24h"]      = p.rolling(window=24, min_periods=1).sum()
    df["Precip_x_Soil"] = p * sm
    df["Soil_lag1"]     = sm.shift(1).fillna(sm.iloc[0])
    df["Soil_lag3"]     = sm.shift(3).fillna(sm.iloc[0])
    df["Soil_rate"]     = sm - df["Soil_lag1"]
    df["Temp_lag1"]     = t.shift(1).fillna(t.iloc[0])

    return df.dropna().reset_index(drop=True)


# ─────────────────────────────────────────────────────────────────────────────
# Public function
# ─────────────────────────────────────────────────────────────────────────────

def predict_24h(lat: float, lon: float) -> dict:
    """Predict flood risk across the next 24 hours for a geographic coordinate.



    Fetches real observed data (past 24h) + NWP forecast (next 24h) from

    Open-Meteo, engineers features across the full 48-row window so lag

    features are grounded in reality, then slides the nowcast LSTM over

    each future timestep.



    Returns the MAXIMUM probability seen across all 24 future steps β€” i.e.

    "what is the chance of flooding at any point in the next 24 hours?"



    Args:

        lat: Latitude  (-90 to 90)

        lon: Longitude (-180 to 180)



    Returns:

        dict with keys:

            lat, lon, flood_probability, alert_level,

            forecast_horizon_hours, based_on_data_until,

            peak_flood_time, features_snapshot, threshold_used

    """
    model, scaler, cfg, threshold = _load_assets()
    device = torch.device("cpu")

    # ── 1. Fetch 24h history + 24h future in one call ─────────────────────────
    raw_df = fetch_forecast_window(lat, lon,
                                   past_hours=PAST_HOURS,
                                   forecast_hours=FORECAST_HOURS)

    # ── 2. Engineer features across ALL 48 rows ───────────────────────────────
    # Stash is_forecast and Timestamp BEFORE engineering so they are not
    # affected by dropna or any other transformation inside _engineer_features.
    is_forecast_arr = raw_df["is_forecast"].values.copy()   # bool[N]
    timestamps_arr  = raw_df["Timestamp"].values.copy()     # datetime[N]

    # Pass only weather columns (no is_forecast) to avoid confusion
    weather_df = raw_df.drop(columns=["is_forecast"])
    feat_df    = _engineer_features(weather_df)

    # _engineer_features may drop leading rows (dropna) β€” align by tail
    n_dropped = len(raw_df) - len(feat_df)
    feat_df = feat_df.copy()
    feat_df["is_forecast"] = is_forecast_arr[n_dropped:]
    feat_df["Timestamp"]   = pd.to_datetime(timestamps_arr[n_dropped:])

    # ── 3. Identify future rows ───────────────────────────────────────────────
    future_mask    = feat_df["is_forecast"].astype(bool)
    future_indices = feat_df.index[future_mask].tolist()

    if not future_indices:
        raise ValueError("No future rows returned by the forecast API.")

    # ── 4. Scale the full feature matrix ─────────────────────────────────────
    X_all    = feat_df[FEATURE_COLUMNS].values.astype(np.float32)
    X_scaled = scaler.transform(X_all).astype(np.float32)

    # ── 5. Slide LSTM window over each future step ────────────────────────────
    # Window ending at future row i  β†’  X_scaled[i-W+1 : i+1]
    # If i < W-1 we'd need rows before the start of feat_df β€” skip those.
    W = NOWCAST_WINDOW
    probs: list[tuple[float, str]] = []   # (probability, timestamp_str)

    model.eval()
    with torch.no_grad():
        timestamps = feat_df["Timestamp"].astype(str).tolist() if "Timestamp" in feat_df.columns else [
            str(raw_df["Timestamp"].iloc[idx]) if idx < len(raw_df) else f"t+{i+1}h"
            for i, idx in enumerate(future_indices)
        ]

        for idx in future_indices:
            start = idx - W + 1
            if start < 0:
                start = 0   # pad from start of available data (first row repeated conceptually)
            window = X_scaled[start: idx + 1]

            # Pad with first row if window is shorter than W (edge case)
            if len(window) < W:
                pad      = np.repeat(window[:1], W - len(window), axis=0)
                window   = np.concatenate([pad, window], axis=0)

            tensor   = torch.from_numpy(window).unsqueeze(0).to(device)  # (1, W, F)
            prob_val = float(model(tensor).item())  # model applies sigmoid internally
            ts_str   = str(feat_df["Timestamp"].iloc[idx]) if "Timestamp" in feat_df.columns else f"t+{idx}h"
            probs.append((prob_val, ts_str))

    # ── 6. Aggregate ──────────────────────────────────────────────────────────
    max_prob, peak_time = max(probs, key=lambda x: x[0])

    # threshold_used stays in the original [0,1] model probability space

    # ── 7. Feature snapshot β€” last future row ─────────────────────────────────
    last_future_idx   = future_indices[-1]
    last_row          = feat_df.iloc[last_future_idx]
    features_snapshot = {col: round(float(last_row[col]), 4) for col in FEATURE_COLUMNS}

    data_until = str(raw_df["Timestamp"].iloc[-1]) if "Timestamp" in raw_df.columns else "unknown"

    return {
        "lat":                    lat,
        "lon":                    lon,
        "flood_probability":      round(max_prob, 6),
        "alert_level":            _alert_level(max_prob, threshold),
        "forecast_horizon_hours": FORECAST_HOURS,
        "based_on_data_until":    data_until,
        "peak_flood_time":        peak_time,
        "features_snapshot":      features_snapshot,
        "threshold_used":         threshold,
    }