""" Directly traces a handful of real false positives (from evaluate_flood_ detection.py's --dump-fp-details output) through build_forecast_tensor's actual real lookup logic, by hand, to check for a real date-convention mismatch -- built after noticing something concrete while investigating the day-5 false-positive pattern: Every other real place in this project that maps a horizon h to a real calendar date uses target_date = anchor + (h - 1) -- confirmed directly in both prepare_graph_training_windows (the real training-target construction) and evaluate_flood_detection.py's own confusion-matrix loop. But build_forecast_tensor uses target_date = anchor + h (no "-1"). This means: for horizon h, the real target this project is actually predicting is at anchor + (h-1), but the forecast value FED INTO the model for that same horizon slot is looked up at anchor + h -- one real calendar day later than the thing actually being predicted. This script shows exactly what real forecast row/column gets retrieved for real (anchor, h=5) pairs pulled from an actual false-positive dump, so the real size and direction of this mismatch is visible directly, not just asserted. Usage: python -m scripts.analysis.trace_forecast_lookup --data-root datasets \ --fp-details datasets/fp_details_horizon_5.csv --horizon 5 """ import argparse from pathlib import Path import pandas as pd def trace_one(anchor: pd.Timestamp, station_code: str, horizon: int, df_indexed: pd.DataFrame) -> dict: """ Mirrors build_forecast_tensor's real lookup exactly -- same target_date/column formula, same real row lookup -- so this shows precisely what that function actually retrieves for this real (anchor, horizon) pair, not an approximation of it. """ real_target_date = anchor + pd.Timedelta(days=horizon - 1) # what's ACTUALLY being predicted forecast_lookup_date = anchor + pd.Timedelta(days=horizon) # what build_forecast_tensor ACTUALLY queries col = f"precipitation_previous_day{horizon}" result = { "anchor": anchor.date().isoformat(), "real_target_date": real_target_date.date().isoformat(), "forecast_lookup_date": forecast_lookup_date.date().isoformat(), "date_mismatch_days": (forecast_lookup_date - real_target_date).days, "station_code": station_code, "column": col, } try: value = df_indexed.loc[(forecast_lookup_date, station_code), col] result["forecast_value_actually_used"] = float(value) if pd.notna(value) else None except KeyError: result["forecast_value_actually_used"] = None # What the forecast value WOULD have been if looked up at the real # target date instead, for direct comparison -- not necessarily a # "correct" value (that date's own previous_day{h} column reflects # a different, (h) days in advance forecast, not h-1), just useful # to see whether it's meaningfully different from what's actually used. try: value_at_real_target = df_indexed.loc[(real_target_date, station_code), col] result["value_at_real_target_date_same_column"] = float(value_at_real_target) if pd.notna(value_at_real_target) else None except KeyError: result["value_at_real_target_date_same_column"] = None return result def main() -> None: parser = argparse.ArgumentParser(description="Trace real forecast lookups for a handful of real false positives") parser.add_argument("--data-root", type=Path, default=Path("datasets")) parser.add_argument("--forecast-path", type=Path, default=None, help="Defaults to /previous_runs_forecast.csv") parser.add_argument("--fp-details", type=Path, required=True, help="Path to a real fp_details_horizon_N.csv from evaluate_flood_detection.py's --dump-fp-details") parser.add_argument("--horizon", type=int, required=True) parser.add_argument("--n-samples", type=int, default=10) args = parser.parse_args() forecast_path = args.forecast_path or (args.data_root / "previous_runs_forecast.csv") if not forecast_path.exists(): print(f"No real forecast archive found at {forecast_path}.") return if not args.fp_details.exists(): print(f"No real false-positive detail file found at {args.fp_details}.") return forecast_df = pd.read_csv(forecast_path) forecast_df["date"] = pd.to_datetime(forecast_df["date"]) df_indexed = forecast_df.set_index(["date", "station_code"]) fp_df = pd.read_csv(args.fp_details) fp_df["target_date"] = pd.to_datetime(fp_df["target_date"]) # fp_details' own target_date IS the real target date (h-1 # convention, matching evaluate_flood_detection.py's own confusion- # matrix loop) -- recover the real anchor from it directly. fp_df["anchor"] = fp_df["target_date"] - pd.Timedelta(days=args.horizon - 1) sample = fp_df.head(args.n_samples) print(f"Tracing {len(sample)} real false positive(s) at horizon={args.horizon}d through the real " f"forecast lookup logic\n") rows = [] for _, row in sample.iterrows(): traced = trace_one(row["anchor"], row["station_code"], args.horizon, df_indexed) rows.append(traced) result_df = pd.DataFrame(rows) print(result_df.to_string(index=False)) n_mismatched_dates = int((result_df["date_mismatch_days"] != 0).sum()) print(f"\n{n_mismatched_dates}/{len(result_df)} real traced examples show a real date mismatch " f"between what's being predicted (real_target_date) and what the forecast is actually " f"looked up for (forecast_lookup_date).") if n_mismatched_dates > 0: print(f"This is a real, systematic {result_df['date_mismatch_days'].iloc[0]}-day offset (from " f"build_forecast_tensor's target_date = anchor + h, vs. every other real place in this " f"project using anchor + h - 1) -- the model is being conditioned on the forecast for the " f"day AFTER what it's actually predicting at this horizon, not the forecast for the real " f"target date itself.") if __name__ == "__main__": main()