BijliCast / src /features /inference.py
muhammadumar15's picture
Deploy full Gradio bundle
20004e3 verified
Raw History Blame Contribute Delete
7.42 kB
"""Inference helpers for BijliCast (train / serve parity).
A deployed app cannot build 24h/168h lags from a single form row. The contract
is therefore:
household profile (static attributes)
+ the last 168 hours of hourly usage for that house
+ current-hour weather
+ the origin timestamp
-> one row in ``INPUT_COLUMNS`` -> the saved sklearn pipeline -> prediction
``build_inference_row`` reproduces exactly the lag/rolling definitions used by
``src.features.build_features``. ``self_check`` proves that parity on real data.
CLI:
python -m src.features.inference
exports ``models/house_profiles.csv`` and runs the parity check.
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pandas as pd
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from src.utils.config import ( # noqa: E402
CITY_COL,
FEATURES_DATASET,
HOUSE_COL,
INPUT_COLUMNS,
INPUT_WEATHER_FEATURES,
LAG_FEATURES,
MERGED_DATASET,
MODELS_DIR,
ROLLING_FEATURES,
SOURCE_USAGE,
SPLITS_DIR,
STATIC_CATEGORICAL_FEATURES,
STATIC_NUMERIC_FEATURES,
TARGET,
TIME_COL,
)
HISTORY_HOURS = 168
PROFILE_COLUMNS = [HOUSE_COL] + STATIC_CATEGORICAL_FEATURES + STATIC_NUMERIC_FEATURES
def export_house_profiles(
merged: pd.DataFrame,
out_path: Path | str | None = None,
) -> tuple[pd.DataFrame, Path]:
"""Write one static-profile row per household (used by all three apps)."""
out_path = Path(out_path or (MODELS_DIR / "house_profiles.csv"))
profiles = (
merged[PROFILE_COLUMNS]
.drop_duplicates(subset=[HOUSE_COL])
.sort_values(HOUSE_COL)
.reset_index(drop=True)
)
out_path.parent.mkdir(parents=True, exist_ok=True)
profiles.to_csv(out_path, index=False)
return profiles, out_path
def build_inference_row(
origin_time,
city: str,
weather: dict,
usage_history: pd.Series,
static: dict,
house_id: str | None = None,
) -> pd.DataFrame:
"""Build one model-ready row from raw deployment inputs.
Parameters
----------
origin_time
Timestamp of the last known hour ``t``; the model predicts ``t+1``.
city
One of the six city names.
weather
Current-hour values for ``INPUT_WEATHER_FEATURES``.
usage_history
Hourly usage Series spanning at least ``origin_time - 167h .. origin_time``.
static
Household profile: the 18 static numeric fields plus ``Owner/Rented``.
"""
origin = pd.Timestamp(origin_time)
missing_weather = [k for k in INPUT_WEATHER_FEATURES if k not in weather]
if missing_weather:
raise ValueError(f"weather is missing: {missing_weather}")
missing_static = [k for k in STATIC_NUMERIC_FEATURES if k not in static]
if missing_static:
raise ValueError(f"static profile is missing: {missing_static}")
hist = pd.Series(usage_history, dtype="float64").copy()
hist.index = pd.to_datetime(hist.index)
hist = hist.sort_index()
stamps = [origin - pd.Timedelta(hours=k) for k in range(HISTORY_HOURS)]
missing_hours = [t for t in stamps if t not in hist.index]
if missing_hours:
raise ValueError(
f"usage_history is missing {len(missing_hours)} of the last "
f"{HISTORY_HOURS} hours (e.g. {missing_hours[:3]})"
)
window = hist.reindex(stamps) # most recent first
if window.isna().any():
raise ValueError("usage_history contains NaN inside the last 168 hours")
row = {
TIME_COL: origin,
"lag_1h": float(window.iloc[0]),
"lag_2h": float(window.iloc[1]),
"lag_3h": float(window.iloc[2]),
"lag_24h": float(window.iloc[23]),
"lag_168h": float(window.iloc[167]),
"rolling_mean_3h": float(window.iloc[:3].mean()),
"rolling_mean_24h": float(window.iloc[:24].mean()),
"rolling_std_24h": float(window.iloc[:24].std()),
}
row.update({k: float(weather[k]) for k in INPUT_WEATHER_FEATURES})
row.update({k: static[k] for k in STATIC_NUMERIC_FEATURES})
row[CITY_COL] = city
row["Owner/Rented"] = static.get("Owner/Rented", np.nan)
frame = pd.DataFrame([row])[INPUT_COLUMNS]
if house_id is not None:
frame.insert(0, HOUSE_COL, house_id)
return frame
def self_check() -> None:
"""Verify the inference row matches the training feature builder."""
merged = pd.read_parquet(MERGED_DATASET)
features = pd.read_parquet(FEATURES_DATASET)
merged[TIME_COL] = pd.to_datetime(merged[TIME_COL])
features[TIME_COL] = pd.to_datetime(features[TIME_COL])
chosen = None
for house, g in merged.groupby(HOUSE_COL):
g = g.sort_values(TIME_COL)
grid = pd.date_range(g[TIME_COL].min(), g[TIME_COL].max(), freq="h")
complete = len(g) == len(grid) and g[SOURCE_USAGE].notna().all()
if complete and g["Air Conditioners"].notna().all():
chosen = house
break
if chosen is None:
raise AssertionError("no clean household found for the parity check")
g = merged[merged[HOUSE_COL] == chosen].sort_values(TIME_COL)
history = g.set_index(TIME_COL)[SOURCE_USAGE]
origin = history.index[300]
first = g.iloc[0]
weather = (
g.loc[g[TIME_COL] == origin, INPUT_WEATHER_FEATURES].iloc[0].to_dict()
)
static = {k: first[k] for k in STATIC_NUMERIC_FEATURES}
static["Owner/Rented"] = first["Owner/Rented"]
row = build_inference_row(
origin, first[CITY_COL], weather, history, static, house_id=chosen
)
expected = features[
(features[HOUSE_COL] == chosen) & (features[TIME_COL] == origin)
]
if len(expected) != 1:
raise AssertionError("could not locate the matching training row")
expected = expected.iloc[0]
numeric = LAG_FEATURES + ROLLING_FEATURES + INPUT_WEATHER_FEATURES
numeric += STATIC_NUMERIC_FEATURES
for col in numeric:
if not np.allclose(row[col].iloc[0], expected[col], equal_nan=True):
raise AssertionError(f"inference/training mismatch in {col!r}")
for col in STATIC_CATEGORICAL_FEATURES:
if row[col].iloc[0] != expected[col]:
raise AssertionError(f"inference/training mismatch in {col!r}")
print(f"parity check passed for {chosen} at {origin}")
train_path = SPLITS_DIR / "train.parquet"
if train_path.exists():
from src.preprocessing.pipeline import build_feature_pipeline, get_feature_names
train = pd.read_parquet(train_path)
prep = build_feature_pipeline()
prep.fit(train[INPUT_COLUMNS])
X = prep.transform(row[INPUT_COLUMNS])
expected_width = len(get_feature_names(prep))
if X.shape != (1, expected_width) or np.isnan(X).any():
raise AssertionError(
f"pipeline transform failed: shape={X.shape}, "
f"expected width={expected_width}"
)
print(f"pipeline accepts the inference row: {X.shape}, no NaNs")
def main() -> None:
merged = pd.read_parquet(MERGED_DATASET)
merged[TIME_COL] = pd.to_datetime(merged[TIME_COL])
profiles, path = export_house_profiles(merged)
print(f"exported {len(profiles)} household profiles -> {path}")
print(f"columns: {list(profiles.columns)}")
self_check()
if __name__ == "__main__":
main()