linear v1 (Unity Catalog workspace.aq.pm25_linear)
Browse files- linear/MLmodel +88 -0
- linear/artifacts/linear/meta.json +57 -0
- linear/artifacts/linear/t24.joblib +3 -0
- linear/artifacts/linear/t48.joblib +3 -0
- linear/artifacts/linear/t72.joblib +3 -0
- linear/code/aq_models/__init__.py +1 -0
- linear/code/aq_models/common.py +79 -0
- linear/code/aq_models/gru.py +198 -0
- linear/code/aq_models/lgbm.py +98 -0
- linear/code/aq_models/linear.py +108 -0
- linear/code/aq_models/pyfunc.py +32 -0
- linear/conda.yaml +11 -0
- linear/metadata/MLmodel +88 -0
- linear/metadata/conda.yaml +11 -0
- linear/metadata/python_env.yaml +7 -0
- linear/metadata/requirements.txt +4 -0
- linear/python_env.yaml +7 -0
- linear/python_model.pkl +3 -0
- linear/requirements.txt +4 -0
linear/MLmodel
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
artifact_path: dbfs:/databricks/mlflow-tracking/50496446126707/logged_models/m-6929babe95a54d199a1e15dce7d95fa3/artifacts
|
| 2 |
+
databricks_runtime: client.4
|
| 3 |
+
flavors:
|
| 4 |
+
python_function:
|
| 5 |
+
artifacts:
|
| 6 |
+
model:
|
| 7 |
+
path: artifacts/linear
|
| 8 |
+
uri: /tmp/tmpk5u_j3kw/linear
|
| 9 |
+
cloudpickle_version: 3.0.0
|
| 10 |
+
code: code
|
| 11 |
+
env:
|
| 12 |
+
conda: conda.yaml
|
| 13 |
+
virtualenv: python_env.yaml
|
| 14 |
+
loader_module: mlflow.pyfunc.model
|
| 15 |
+
python_model: python_model.pkl
|
| 16 |
+
python_version: 3.12.3
|
| 17 |
+
streamable: false
|
| 18 |
+
is_signature_from_type_hint: false
|
| 19 |
+
mlflow_version: 3.18.0
|
| 20 |
+
model_id: m-6929babe95a54d199a1e15dce7d95fa3
|
| 21 |
+
model_size_bytes: 6941
|
| 22 |
+
model_uuid: m-6929babe95a54d199a1e15dce7d95fa3
|
| 23 |
+
prompts: null
|
| 24 |
+
run_id: 72c567f94a134640953c4ee266d4655d
|
| 25 |
+
signature:
|
| 26 |
+
inputs: '[{"type": "string", "name": "station_id", "required": true}, {"type": "datetime",
|
| 27 |
+
"name": "hour_utc", "required": true}, {"type": "double", "name": "latitude",
|
| 28 |
+
"required": true}, {"type": "double", "name": "longitude", "required": true},
|
| 29 |
+
{"type": "boolean", "name": "wasatch_front", "required": true}, {"type": "double",
|
| 30 |
+
"name": "pm25_ugm3", "required": false}, {"type": "double", "name": "no2_ppb",
|
| 31 |
+
"required": false}, {"type": "double", "name": "pm25_lag_1h", "required": false},
|
| 32 |
+
{"type": "double", "name": "pm25_lag_2h", "required": false}, {"type": "double",
|
| 33 |
+
"name": "pm25_lag_3h", "required": false}, {"type": "double", "name": "pm25_lag_6h",
|
| 34 |
+
"required": false}, {"type": "double", "name": "pm25_lag_12h", "required": false},
|
| 35 |
+
{"type": "double", "name": "pm25_lag_24h", "required": false}, {"type": "double",
|
| 36 |
+
"name": "pm25_lag_48h", "required": true}, {"type": "double", "name": "pm25_lag_72h",
|
| 37 |
+
"required": false}, {"type": "double", "name": "pm25_mean_3h", "required": true},
|
| 38 |
+
{"type": "double", "name": "pm25_mean_6h", "required": true}, {"type": "double",
|
| 39 |
+
"name": "pm25_mean_12h", "required": true}, {"type": "double", "name": "pm25_mean_24h",
|
| 40 |
+
"required": true}, {"type": "double", "name": "pm25_mean_72h", "required": true},
|
| 41 |
+
{"type": "double", "name": "pm25_mean_168h", "required": true}, {"type": "double",
|
| 42 |
+
"name": "pm25_max_24h", "required": true}, {"type": "double", "name": "pm25_std_24h",
|
| 43 |
+
"required": true}, {"type": "long", "name": "pm25_obs_24h", "required": true},
|
| 44 |
+
{"type": "double", "name": "pm25_change_24h", "required": false}, {"type": "double",
|
| 45 |
+
"name": "no2_mean_24h", "required": true}, {"type": "double", "name": "region_pm25",
|
| 46 |
+
"required": true}, {"type": "double", "name": "region_pm25_mean_24h", "required":
|
| 47 |
+
true}, {"type": "integer", "name": "hour_local", "required": true}, {"type": "integer",
|
| 48 |
+
"name": "dow_local", "required": true}, {"type": "integer", "name": "doy_local",
|
| 49 |
+
"required": true}, {"type": "boolean", "name": "fireworks_t24", "required": true},
|
| 50 |
+
{"type": "boolean", "name": "fireworks_t48", "required": true}, {"type": "boolean",
|
| 51 |
+
"name": "fireworks_t72", "required": true}, {"type": "double", "name": "wx24_temp_c",
|
| 52 |
+
"required": false}, {"type": "double", "name": "wx24_rh_pct", "required": false},
|
| 53 |
+
{"type": "double", "name": "wx24_wind_speed_ms", "required": false}, {"type":
|
| 54 |
+
"double", "name": "wx24_wind_dir_deg", "required": false}, {"type": "double",
|
| 55 |
+
"name": "wx24_cloud_cover_pct", "required": false}, {"type": "double", "name":
|
| 56 |
+
"wx24_shortwave_wm2", "required": false}, {"type": "double", "name": "wx24_inversion_c",
|
| 57 |
+
"required": false}, {"type": "double", "name": "wx24_wind_mean_24h", "required":
|
| 58 |
+
false}, {"type": "double", "name": "wx24_wind_max_24h", "required": false}, {"type":
|
| 59 |
+
"double", "name": "wx24_precip_24h_mm", "required": false}, {"type": "double",
|
| 60 |
+
"name": "wx24_inversion_mean_24h", "required": false}, {"type": "double", "name":
|
| 61 |
+
"wx24_pressure_change_24h_hpa", "required": false}, {"type": "double", "name":
|
| 62 |
+
"wx48_temp_c", "required": false}, {"type": "double", "name": "wx48_rh_pct", "required":
|
| 63 |
+
false}, {"type": "double", "name": "wx48_wind_speed_ms", "required": false}, {"type":
|
| 64 |
+
"double", "name": "wx48_wind_dir_deg", "required": false}, {"type": "double",
|
| 65 |
+
"name": "wx48_cloud_cover_pct", "required": false}, {"type": "double", "name":
|
| 66 |
+
"wx48_shortwave_wm2", "required": false}, {"type": "double", "name": "wx48_inversion_c",
|
| 67 |
+
"required": false}, {"type": "double", "name": "wx48_wind_mean_24h", "required":
|
| 68 |
+
false}, {"type": "double", "name": "wx48_wind_max_24h", "required": false}, {"type":
|
| 69 |
+
"double", "name": "wx48_precip_24h_mm", "required": false}, {"type": "double",
|
| 70 |
+
"name": "wx48_inversion_mean_24h", "required": false}, {"type": "double", "name":
|
| 71 |
+
"wx48_pressure_change_24h_hpa", "required": false}, {"type": "double", "name":
|
| 72 |
+
"wx72_temp_c", "required": false}, {"type": "double", "name": "wx72_rh_pct", "required":
|
| 73 |
+
false}, {"type": "double", "name": "wx72_wind_speed_ms", "required": false}, {"type":
|
| 74 |
+
"double", "name": "wx72_wind_dir_deg", "required": false}, {"type": "double",
|
| 75 |
+
"name": "wx72_cloud_cover_pct", "required": false}, {"type": "double", "name":
|
| 76 |
+
"wx72_shortwave_wm2", "required": false}, {"type": "double", "name": "wx72_inversion_c",
|
| 77 |
+
"required": false}, {"type": "double", "name": "wx72_wind_mean_24h", "required":
|
| 78 |
+
false}, {"type": "double", "name": "wx72_wind_max_24h", "required": false}, {"type":
|
| 79 |
+
"double", "name": "wx72_precip_24h_mm", "required": false}, {"type": "double",
|
| 80 |
+
"name": "wx72_inversion_mean_24h", "required": false}, {"type": "double", "name":
|
| 81 |
+
"wx72_pressure_change_24h_hpa", "required": false}]'
|
| 82 |
+
outputs: '[{"type": "string", "name": "station_id", "required": true}, {"type":
|
| 83 |
+
"datetime", "name": "hour_utc", "required": true}, {"type": "double", "name":
|
| 84 |
+
"pred_t24", "required": true}, {"type": "double", "name": "pred_t48", "required":
|
| 85 |
+
true}, {"type": "double", "name": "pred_t72", "required": true}]'
|
| 86 |
+
params: null
|
| 87 |
+
type_hint_from_example: false
|
| 88 |
+
utc_time_created: '2026-11-05 14:26:51.280529'
|
linear/artifacts/linear/meta.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cfg": {
|
| 3 |
+
"variant": "small",
|
| 4 |
+
"alpha": 1.0
|
| 5 |
+
},
|
| 6 |
+
"stations": [
|
| 7 |
+
"beach",
|
| 8 |
+
"bountiful",
|
| 9 |
+
"brigham_city",
|
| 10 |
+
"copper_view",
|
| 11 |
+
"enoch",
|
| 12 |
+
"erda",
|
| 13 |
+
"harrisville",
|
| 14 |
+
"hawthorne",
|
| 15 |
+
"heber",
|
| 16 |
+
"herriman",
|
| 17 |
+
"hurricane",
|
| 18 |
+
"lake_park",
|
| 19 |
+
"lindon",
|
| 20 |
+
"magna",
|
| 21 |
+
"moab",
|
| 22 |
+
"near_road",
|
| 23 |
+
"price",
|
| 24 |
+
"prison_site",
|
| 25 |
+
"red_butte",
|
| 26 |
+
"roosevelt",
|
| 27 |
+
"rose_park",
|
| 28 |
+
"smithfield",
|
| 29 |
+
"spanish_fork",
|
| 30 |
+
"utah_tech_center",
|
| 31 |
+
"vernal",
|
| 32 |
+
"washakie"
|
| 33 |
+
],
|
| 34 |
+
"features": {
|
| 35 |
+
"24": [
|
| 36 |
+
"pm25_ugm3",
|
| 37 |
+
"pm25_mean_24h",
|
| 38 |
+
"wx24_inversion_mean_24h",
|
| 39 |
+
"wx24_wind_mean_24h",
|
| 40 |
+
"wx24_precip_24h_mm"
|
| 41 |
+
],
|
| 42 |
+
"48": [
|
| 43 |
+
"pm25_ugm3",
|
| 44 |
+
"pm25_mean_24h",
|
| 45 |
+
"wx48_inversion_mean_24h",
|
| 46 |
+
"wx48_wind_mean_24h",
|
| 47 |
+
"wx48_precip_24h_mm"
|
| 48 |
+
],
|
| 49 |
+
"72": [
|
| 50 |
+
"pm25_ugm3",
|
| 51 |
+
"pm25_mean_24h",
|
| 52 |
+
"wx72_inversion_mean_24h",
|
| 53 |
+
"wx72_wind_mean_24h",
|
| 54 |
+
"wx72_precip_24h_mm"
|
| 55 |
+
]
|
| 56 |
+
}
|
| 57 |
+
}
|
linear/artifacts/linear/t24.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:08fd8a6df74bd42b216207bdd08c1e23f343be3e2ec1cb1fe4c44ac84913da08
|
| 3 |
+
size 1937
|
linear/artifacts/linear/t48.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e74d85476ec91a41425c9b47009f3d90ac6dbc85adf079c26c642241c5edaca7
|
| 3 |
+
size 1937
|
linear/artifacts/linear/t72.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:223bb35d7e01f751bfa100b15326c42e65e82a61be5b528dd2421eff4a069f4e
|
| 3 |
+
size 1937
|
linear/code/aq_models/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""PM2.5 forecasting models. Shared by local development (training/) and the Databricks training job."""
|
linear/code/aq_models/common.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""What every model shares: which gold columns are features, how to split by time, baselines, and metrics.
|
| 2 |
+
|
| 3 |
+
Each model takes a pandas DataFrame of `gold_features` rows and returns one prediction column per horizon,
|
| 4 |
+
`pred_t24`, `pred_t48`, `pred_t72`, aligned to the input rows.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import pandas as pd
|
| 9 |
+
|
| 10 |
+
HORIZONS = [24, 48, 72]
|
| 11 |
+
LABELS = [f"pm25_t{h}" for h in HORIZONS]
|
| 12 |
+
PREDS = [f"pred_t{h}" for h in HORIZONS]
|
| 13 |
+
KEYS = ["station_id", "hour_utc"]
|
| 14 |
+
MAX_HORIZON = pd.Timedelta(hours=max(HORIZONS))
|
| 15 |
+
|
| 16 |
+
# AQI breakpoints for PM2.5 (EPA, 2024). Applied here to hourly values, so only a rough guide to the daily index.
|
| 17 |
+
UNHEALTHY_SENSITIVE = 35.5
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def feature_columns(df):
|
| 21 |
+
"""Every gold column except the keys and labels. `station_id` goes back in as a categorical feature."""
|
| 22 |
+
return [c for c in df.columns if c not in KEYS and c not in LABELS]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def time_split(df, cutoff, test_days=None):
|
| 26 |
+
"""Train on rows whose labels all fall before `cutoff`, test on rows from `cutoff` on.
|
| 27 |
+
|
| 28 |
+
A row at hour t has labels up to t + 72 h. Training rows within 72 hours of the cutoff would have labels
|
| 29 |
+
inside the test period, so they're dropped (a purge gap). A random split would leak far worse: neighbouring
|
| 30 |
+
hours are nearly identical, so the test set would be mostly copies of training rows.
|
| 31 |
+
"""
|
| 32 |
+
cutoff = pd.Timestamp(cutoff)
|
| 33 |
+
train = df[df["hour_utc"] < cutoff - MAX_HORIZON]
|
| 34 |
+
test = df[df["hour_utc"] >= cutoff]
|
| 35 |
+
if test_days:
|
| 36 |
+
test = test[test["hour_utc"] < cutoff + pd.Timedelta(days=test_days)]
|
| 37 |
+
return train, test
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def baselines(df):
|
| 41 |
+
"""Persistence (the latest reading, for every horizon) and the trailing 24-hour mean."""
|
| 42 |
+
return {
|
| 43 |
+
"persistence": pd.DataFrame({p: df["pm25_ugm3"] for p in PREDS}, index=df.index),
|
| 44 |
+
"mean_24h": pd.DataFrame({p: df["pm25_mean_24h"] for p in PREDS}, index=df.index),
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def season(ts):
|
| 49 |
+
return pd.Series(np.select(
|
| 50 |
+
[ts.dt.month.isin([12, 1, 2]), ts.dt.month.isin([6, 7, 8])], ["winter", "summer"], "spring_fall"
|
| 51 |
+
), index=ts.index)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def evaluate(df, predictions):
|
| 55 |
+
"""MAE / RMSE per model, horizon and subset.
|
| 56 |
+
|
| 57 |
+
Every model is scored on the same rows: label present and a current reading present (so persistence is
|
| 58 |
+
defined). Subsets: all stations, Wasatch Front, by season, and hours at or above the "unhealthy for
|
| 59 |
+
sensitive groups" level, where a forecast matters most.
|
| 60 |
+
"""
|
| 61 |
+
rows = []
|
| 62 |
+
seasons = season(df["hour_utc"])
|
| 63 |
+
for h, label, pred in zip(HORIZONS, LABELS, PREDS):
|
| 64 |
+
base = df[label].notna() & df["pm25_ugm3"].notna()
|
| 65 |
+
subsets = {
|
| 66 |
+
"all": base,
|
| 67 |
+
"wasatch_front": base & df["wasatch_front"].astype(bool),
|
| 68 |
+
"winter": base & (seasons == "winter"),
|
| 69 |
+
"summer": base & (seasons == "summer"),
|
| 70 |
+
f">= {UNHEALTHY_SENSITIVE}": base & (df[label] >= UNHEALTHY_SENSITIVE),
|
| 71 |
+
}
|
| 72 |
+
for model, p in predictions.items():
|
| 73 |
+
for subset, mask in subsets.items():
|
| 74 |
+
mask = mask & p[pred].notna()
|
| 75 |
+
err = p.loc[mask, pred] - df.loc[mask, label]
|
| 76 |
+
rows.append({"model": model, "horizon_h": h, "subset": subset, "n": int(mask.sum()),
|
| 77 |
+
"mae": float(err.abs().mean()), "rmse": float(np.sqrt((err ** 2).mean())),
|
| 78 |
+
"bias": float(err.mean())})
|
| 79 |
+
return pd.DataFrame(rows)
|
linear/code/aq_models/gru.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GRU, trained from scratch: the last 72 hours as a sequence, plus the forecast weather, with three outputs.
|
| 2 |
+
|
| 3 |
+
last 72 h of [PM2.5, missing flag, NO₂, regional PM2.5, hour of day] ──► GRU ──┐
|
| 4 |
+
weather forecasts for each horizon, calendar, rolling stats (standardized) ────┼──► MLP ──► Δ for t+24/48/72
|
| 5 |
+
station embedding ─────────────────────────────────────────────────────────────┘
|
| 6 |
+
|
| 7 |
+
The network predicts the *change* from the current reading, not the level. Persistence is then the starting
|
| 8 |
+
point, and the network only has to learn when PM2.5 will move away from it.
|
| 9 |
+
|
| 10 |
+
The sequence is read from consecutive gold rows. Gold has a row for every station-hour, so 72 rows are
|
| 11 |
+
exactly 72 hours. A row gets a prediction only if the 71 rows before it are the same station's previous hours.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
import pandas as pd
|
| 19 |
+
import torch
|
| 20 |
+
from torch import nn
|
| 21 |
+
|
| 22 |
+
from .common import HORIZONS, KEYS, LABELS, PREDS, feature_columns
|
| 23 |
+
|
| 24 |
+
SEQ_LEN = 72
|
| 25 |
+
SEQ_COLS = ["pm25_ugm3", "no2_ppb", "region_pm25"]
|
| 26 |
+
SCALE = 10.0 # µg/m³ per unit of network output
|
| 27 |
+
VALID_DAYS = 28
|
| 28 |
+
CONFIG = {"hidden": 64, "head": 128, "dropout": 0.1, "embedding": 8, "lr": 2e-3, "batch": 1024,
|
| 29 |
+
"max_epochs": 20, "patience": 3, "seed": 7}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class Net(nn.Module):
|
| 33 |
+
def __init__(self, n_seq, n_static, n_stations, cfg):
|
| 34 |
+
super().__init__()
|
| 35 |
+
self.embedding = nn.Embedding(n_stations + 1, cfg["embedding"]) # last index: a station not seen in training
|
| 36 |
+
self.gru = nn.GRU(n_seq, cfg["hidden"], batch_first=True)
|
| 37 |
+
self.head = nn.Sequential(
|
| 38 |
+
nn.Linear(cfg["hidden"] + n_static + cfg["embedding"], cfg["head"]), nn.ReLU(), nn.Dropout(cfg["dropout"]),
|
| 39 |
+
nn.Linear(cfg["head"], len(HORIZONS)),
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
def forward(self, seq, static, station):
|
| 43 |
+
_, h = self.gru(seq)
|
| 44 |
+
return self.head(torch.cat([h[-1], static, self.embedding(station)], dim=1))
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _device():
|
| 48 |
+
return torch.device("mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class GRUForecaster:
|
| 52 |
+
def __init__(self, cfg=None):
|
| 53 |
+
self.cfg = {**CONFIG, **(cfg or {})}
|
| 54 |
+
self.net = None
|
| 55 |
+
self.static_cols = None
|
| 56 |
+
self.mean = self.std = None
|
| 57 |
+
self.stations = None
|
| 58 |
+
self.epochs_trained = 0
|
| 59 |
+
|
| 60 |
+
# --- inputs -------------------------------------------------------------------------------------------------
|
| 61 |
+
|
| 62 |
+
def _arrays(self, df):
|
| 63 |
+
"""Sort by station and hour, then build the per-hour sequence channels, static features, and anchors."""
|
| 64 |
+
df = df.sort_values(KEYS)
|
| 65 |
+
log = lambda c: np.log1p(df[c].clip(lower=0).to_numpy(dtype="float32")) # noqa: E731
|
| 66 |
+
pm25 = df["pm25_ugm3"].to_numpy(dtype="float32")
|
| 67 |
+
hour = df["hour_local"].to_numpy(dtype="float32") * (2 * np.pi / 24)
|
| 68 |
+
seq = np.stack([log("pm25_ugm3"), np.isnan(pm25).astype("float32"), log("no2_ppb"), log("region_pm25"),
|
| 69 |
+
np.sin(hour), np.cos(hour)], axis=1)
|
| 70 |
+
seq = np.nan_to_num(seq)
|
| 71 |
+
|
| 72 |
+
static = df[self.static_cols].astype("float32").to_numpy()
|
| 73 |
+
static = np.nan_to_num((static - self.mean) / self.std) # missing → the training mean
|
| 74 |
+
index = {s: i for i, s in enumerate(self.stations)}
|
| 75 |
+
station = df["station_id"].map(index).fillna(len(self.stations)).to_numpy(dtype="int64")
|
| 76 |
+
|
| 77 |
+
anchor = df["pm25_ugm3"].fillna(df["pm25_mean_24h"]).to_numpy(dtype="float32") / SCALE
|
| 78 |
+
start = df.groupby("station_id")["hour_utc"].shift(SEQ_LEN - 1)
|
| 79 |
+
usable = ((df["hour_utc"] - start) == pd.Timedelta(hours=SEQ_LEN - 1)).to_numpy() & ~np.isnan(anchor)
|
| 80 |
+
return df, seq, static, station, anchor, usable
|
| 81 |
+
|
| 82 |
+
def _static_columns(self, df):
|
| 83 |
+
cols = [c for c in feature_columns(df) if c not in ["station_id", *SEQ_COLS]]
|
| 84 |
+
return [c for c in cols if pd.api.types.is_numeric_dtype(df[c]) or pd.api.types.is_bool_dtype(df[c])]
|
| 85 |
+
|
| 86 |
+
@staticmethod
|
| 87 |
+
def _windows(seq, idx):
|
| 88 |
+
return seq[idx[:, None] + np.arange(-(SEQ_LEN - 1), 1)]
|
| 89 |
+
|
| 90 |
+
def _batches(self, arrays, idx, targets=None, shuffle=False):
|
| 91 |
+
_, seq, static, station, anchor, _ = arrays
|
| 92 |
+
if shuffle:
|
| 93 |
+
idx = np.random.permutation(idx)
|
| 94 |
+
dev = _device()
|
| 95 |
+
for i in range(0, len(idx), self.cfg["batch"]):
|
| 96 |
+
b = idx[i:i + self.cfg["batch"]]
|
| 97 |
+
out = [torch.from_numpy(self._windows(seq, b)).to(dev), torch.from_numpy(static[b]).to(dev),
|
| 98 |
+
torch.from_numpy(station[b]).to(dev), torch.from_numpy(anchor[b]).to(dev)]
|
| 99 |
+
if targets is not None:
|
| 100 |
+
out.append(torch.from_numpy(targets[b]).to(dev))
|
| 101 |
+
yield out
|
| 102 |
+
|
| 103 |
+
def _forward(self, seq, static, station, anchor):
|
| 104 |
+
return anchor[:, None] + self.net(seq, static, station)
|
| 105 |
+
|
| 106 |
+
# --- training -----------------------------------------------------------------------------------------------
|
| 107 |
+
|
| 108 |
+
def fit(self, train, log=print):
|
| 109 |
+
torch.manual_seed(self.cfg["seed"])
|
| 110 |
+
np.random.seed(self.cfg["seed"])
|
| 111 |
+
self.stations = sorted(train["station_id"].unique())
|
| 112 |
+
self.static_cols = self._static_columns(train)
|
| 113 |
+
values = train[self.static_cols].astype("float32")
|
| 114 |
+
self.mean = values.mean().to_numpy(dtype="float32")
|
| 115 |
+
self.std = values.std().replace(0, 1).fillna(1).to_numpy(dtype="float32")
|
| 116 |
+
|
| 117 |
+
arrays = self._arrays(train)
|
| 118 |
+
df, usable = arrays[0], arrays[-1]
|
| 119 |
+
targets = (df[LABELS].to_numpy(dtype="float32") / SCALE)
|
| 120 |
+
has_label = ~np.isnan(targets).all(axis=1)
|
| 121 |
+
split = df["hour_utc"].max() - pd.Timedelta(days=VALID_DAYS)
|
| 122 |
+
late = (df["hour_utc"] >= split).to_numpy()
|
| 123 |
+
fit_idx = np.flatnonzero(usable & has_label & ~late)
|
| 124 |
+
valid_idx = np.flatnonzero(usable & has_label & late)
|
| 125 |
+
|
| 126 |
+
dev = _device()
|
| 127 |
+
self.net = Net(arrays[1].shape[1], arrays[2].shape[1], len(self.stations), self.cfg).to(dev)
|
| 128 |
+
opt = torch.optim.AdamW(self.net.parameters(), lr=self.cfg["lr"], weight_decay=1e-4)
|
| 129 |
+
best, best_state, waited = np.inf, None, 0
|
| 130 |
+
log(f"GRU: {len(fit_idx):,} training sequences, {len(valid_idx):,} validation, device {dev}")
|
| 131 |
+
for epoch in range(1, self.cfg["max_epochs"] + 1):
|
| 132 |
+
self.net.train()
|
| 133 |
+
total = 0.0
|
| 134 |
+
for seq, static, station, anchor, y in self._batches(arrays, fit_idx, targets, shuffle=True):
|
| 135 |
+
pred = self._forward(seq, static, station, anchor)
|
| 136 |
+
mask = ~torch.isnan(y)
|
| 137 |
+
loss = nn.functional.smooth_l1_loss(pred[mask], y[mask], beta=0.5)
|
| 138 |
+
opt.zero_grad()
|
| 139 |
+
loss.backward()
|
| 140 |
+
nn.utils.clip_grad_norm_(self.net.parameters(), 1.0)
|
| 141 |
+
opt.step()
|
| 142 |
+
total += loss.item() * len(y)
|
| 143 |
+
mae = self._mae(arrays, valid_idx, targets)
|
| 144 |
+
log(f" epoch {epoch:2d}: train loss {total / len(fit_idx):.4f}, validation MAE {mae:.3f} µg/m³")
|
| 145 |
+
if mae < best - 1e-3:
|
| 146 |
+
best, waited = mae, 0
|
| 147 |
+
best_state = {k: v.detach().cpu().clone() for k, v in self.net.state_dict().items()}
|
| 148 |
+
self.epochs_trained = epoch
|
| 149 |
+
else:
|
| 150 |
+
waited += 1
|
| 151 |
+
if waited >= self.cfg["patience"]:
|
| 152 |
+
break
|
| 153 |
+
self.net.load_state_dict(best_state)
|
| 154 |
+
return self
|
| 155 |
+
|
| 156 |
+
@torch.no_grad()
|
| 157 |
+
def _mae(self, arrays, idx, targets):
|
| 158 |
+
self.net.eval()
|
| 159 |
+
errs = []
|
| 160 |
+
for seq, static, station, anchor, y in self._batches(arrays, idx, targets):
|
| 161 |
+
e = (self._forward(seq, static, station, anchor) - y) * SCALE
|
| 162 |
+
errs.append(e[~torch.isnan(e)].abs().cpu())
|
| 163 |
+
return torch.cat(errs).mean().item()
|
| 164 |
+
|
| 165 |
+
# --- inference ----------------------------------------------------------------------------------------------
|
| 166 |
+
|
| 167 |
+
@torch.no_grad()
|
| 168 |
+
def predict(self, df):
|
| 169 |
+
"""Predictions aligned to `df`'s index. NaN for rows without 72 consecutive hours of history in `df`."""
|
| 170 |
+
self.net.eval()
|
| 171 |
+
arrays = self._arrays(df)
|
| 172 |
+
sorted_df, usable = arrays[0], arrays[-1]
|
| 173 |
+
idx = np.flatnonzero(usable)
|
| 174 |
+
out = np.full((len(sorted_df), len(HORIZONS)), np.nan, dtype="float32")
|
| 175 |
+
preds = [self._forward(*b).cpu().numpy() for b in self._batches(arrays, idx)]
|
| 176 |
+
if preds:
|
| 177 |
+
out[idx] = np.concatenate(preds) * SCALE
|
| 178 |
+
return pd.DataFrame(out, index=sorted_df.index, columns=PREDS).reindex(df.index)
|
| 179 |
+
|
| 180 |
+
def save(self, path):
|
| 181 |
+
path = Path(path)
|
| 182 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 183 |
+
torch.save(self.net.state_dict(), path / "net.pt")
|
| 184 |
+
(path / "meta.json").write_text(json.dumps({
|
| 185 |
+
"cfg": self.cfg, "static_cols": self.static_cols, "mean": self.mean.tolist(), "std": self.std.tolist(),
|
| 186 |
+
"stations": self.stations, "epochs_trained": self.epochs_trained, "n_seq": 6,
|
| 187 |
+
}, indent=2))
|
| 188 |
+
|
| 189 |
+
@classmethod
|
| 190 |
+
def load(cls, path):
|
| 191 |
+
path = Path(path)
|
| 192 |
+
meta = json.loads((path / "meta.json").read_text())
|
| 193 |
+
m = cls(meta["cfg"])
|
| 194 |
+
m.static_cols, m.stations, m.epochs_trained = meta["static_cols"], meta["stations"], meta["epochs_trained"]
|
| 195 |
+
m.mean, m.std = np.array(meta["mean"], dtype="float32"), np.array(meta["std"], dtype="float32")
|
| 196 |
+
m.net = Net(meta["n_seq"], len(m.static_cols), len(m.stations), m.cfg).to(_device())
|
| 197 |
+
m.net.load_state_dict(torch.load(path / "net.pt", map_location=_device()))
|
| 198 |
+
return m
|
linear/code/aq_models/lgbm.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""LightGBM: one gradient-boosted tree model per horizon, on the gold features as they are.
|
| 2 |
+
|
| 3 |
+
Trees handle NULLs, mixed scales, and interactions (an inversion matters more in January than in July)
|
| 4 |
+
without any preprocessing, so the features go in unchanged. `station_id` is a categorical feature with a
|
| 5 |
+
fixed category list, so codes are the same at training and serving time.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import lightgbm as lgb
|
| 12 |
+
import pandas as pd
|
| 13 |
+
|
| 14 |
+
from .common import HORIZONS, LABELS, PREDS, feature_columns
|
| 15 |
+
|
| 16 |
+
# Tweedie suits a skewed, non-negative target. In the 2025-11 → 2026-10 backtest it beat squared error
|
| 17 |
+
# (lower MAE at every horizon, bias near zero) and Huber at 48/72 h and in winter. No objective fixed the
|
| 18 |
+
# spikes: every variant underpredicts hours ≥ 35.5 µg/m³ by ~33 (see docs/design.md §11).
|
| 19 |
+
PARAMS = {
|
| 20 |
+
"objective": "tweedie",
|
| 21 |
+
"tweedie_variance_power": 1.3,
|
| 22 |
+
"learning_rate": 0.05,
|
| 23 |
+
"num_leaves": 63,
|
| 24 |
+
"min_data_in_leaf": 100,
|
| 25 |
+
"feature_fraction": 0.8,
|
| 26 |
+
"bagging_fraction": 0.8,
|
| 27 |
+
"bagging_freq": 1,
|
| 28 |
+
"lambda_l2": 1.0,
|
| 29 |
+
"verbose": -1,
|
| 30 |
+
}
|
| 31 |
+
VALID_DAYS = 28
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class LightGBMForecaster:
|
| 35 |
+
def __init__(self, params=None):
|
| 36 |
+
self.params = {**PARAMS, **(params or {})}
|
| 37 |
+
self.boosters = {}
|
| 38 |
+
self.features = None
|
| 39 |
+
self.stations = None
|
| 40 |
+
self.best_iterations = {}
|
| 41 |
+
|
| 42 |
+
def _matrix(self, df):
|
| 43 |
+
X = df[self.features].copy()
|
| 44 |
+
X["station_id"] = pd.Categorical(X["station_id"], categories=self.stations)
|
| 45 |
+
for c in X.columns:
|
| 46 |
+
if X[c].dtype == bool or str(X[c].dtype) == "boolean":
|
| 47 |
+
X[c] = X[c].astype("float32")
|
| 48 |
+
return X
|
| 49 |
+
|
| 50 |
+
def fit(self, train):
|
| 51 |
+
"""Early-stop on the last 28 days of the training period, then refit on all of it with that many rounds.
|
| 52 |
+
|
| 53 |
+
The validation window comes from the end of training, never from the test period, so the test set is
|
| 54 |
+
only ever used once, for the final score.
|
| 55 |
+
"""
|
| 56 |
+
self.features = ["station_id"] + feature_columns(train)
|
| 57 |
+
self.stations = sorted(train["station_id"].unique())
|
| 58 |
+
split = train["hour_utc"].max() - pd.Timedelta(days=VALID_DAYS)
|
| 59 |
+
for h, label in zip(HORIZONS, LABELS):
|
| 60 |
+
# Readings can be slightly negative (instrument noise near zero); Tweedie needs labels ≥ 0.
|
| 61 |
+
rows = train[train[label].notna()].assign(**{label: lambda d: d[label].clip(lower=0)})
|
| 62 |
+
fit, valid = rows[rows["hour_utc"] < split], rows[rows["hour_utc"] >= split]
|
| 63 |
+
probe = lgb.train(
|
| 64 |
+
self.params, lgb.Dataset(self._matrix(fit), fit[label]), num_boost_round=3000,
|
| 65 |
+
valid_sets=[lgb.Dataset(self._matrix(valid), valid[label])],
|
| 66 |
+
callbacks=[lgb.early_stopping(100, verbose=False)],
|
| 67 |
+
)
|
| 68 |
+
self.best_iterations[h] = probe.best_iteration
|
| 69 |
+
self.boosters[h] = lgb.train(self.params, lgb.Dataset(self._matrix(rows), rows[label]),
|
| 70 |
+
num_boost_round=probe.best_iteration)
|
| 71 |
+
return self
|
| 72 |
+
|
| 73 |
+
def predict(self, df):
|
| 74 |
+
X = self._matrix(df)
|
| 75 |
+
return pd.DataFrame({p: self.boosters[h].predict(X) for h, p in zip(HORIZONS, PREDS)}, index=df.index)
|
| 76 |
+
|
| 77 |
+
def feature_importance(self, top=15):
|
| 78 |
+
gain = pd.DataFrame({h: b.feature_importance("gain") for h, b in self.boosters.items()}, index=self.features)
|
| 79 |
+
return (gain / gain.sum()).sort_values(HORIZONS[0], ascending=False).head(top)
|
| 80 |
+
|
| 81 |
+
def save(self, path):
|
| 82 |
+
path = Path(path)
|
| 83 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 84 |
+
for h, b in self.boosters.items():
|
| 85 |
+
b.save_model(str(path / f"t{h}.txt"))
|
| 86 |
+
(path / "meta.json").write_text(json.dumps({
|
| 87 |
+
"params": self.params, "features": self.features, "stations": self.stations,
|
| 88 |
+
"best_iterations": self.best_iterations,
|
| 89 |
+
}, indent=2))
|
| 90 |
+
|
| 91 |
+
@classmethod
|
| 92 |
+
def load(cls, path):
|
| 93 |
+
path = Path(path)
|
| 94 |
+
meta = json.loads((path / "meta.json").read_text())
|
| 95 |
+
m = cls(meta["params"])
|
| 96 |
+
m.features, m.stations, m.best_iterations = meta["features"], meta["stations"], {int(k): v for k, v in meta["best_iterations"].items()}
|
| 97 |
+
m.boosters = {h: lgb.Booster(model_file=str(path / f"t{h}.txt")) for h in HORIZONS}
|
| 98 |
+
return m
|
linear/code/aq_models/linear.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Ridge regression: one linear model per horizon, the simplest model that uses weather forecasts.
|
| 2 |
+
|
| 3 |
+
Two variants, to measure what the extra features buy:
|
| 4 |
+
|
| 5 |
+
- `small`: five inputs per horizon. The current reading and the 24-hour mean (the persistence signal), plus
|
| 6 |
+
the forecast inversion strength, wind, and precipitation for the 24 hours ending at the target hour.
|
| 7 |
+
- `full`: every gold feature, with the station one-hot encoded and hour, day of year, and wind direction as
|
| 8 |
+
sin/cos pairs (a linear model can't use an angle or a station code directly).
|
| 9 |
+
|
| 10 |
+
PM2.5 is skewed, so the target and the PM2.5 inputs are log1p-transformed: the model is linear in log space,
|
| 11 |
+
which makes a weather effect multiplicative (an inversion doubles a high reading's growth, not a low one's).
|
| 12 |
+
Missing inputs are filled with the training median, plus an indicator column. Features are standardized, so
|
| 13 |
+
the coefficients are comparable: `coefficients()` shows them for the write-up.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
import joblib
|
| 20 |
+
import numpy as np
|
| 21 |
+
import pandas as pd
|
| 22 |
+
from sklearn.impute import SimpleImputer
|
| 23 |
+
from sklearn.linear_model import Ridge
|
| 24 |
+
from sklearn.pipeline import make_pipeline
|
| 25 |
+
from sklearn.preprocessing import StandardScaler
|
| 26 |
+
|
| 27 |
+
from .common import HORIZONS, LABELS, PREDS, feature_columns
|
| 28 |
+
|
| 29 |
+
ALPHA = 1.0
|
| 30 |
+
SMALL = ["pm25_ugm3", "pm25_mean_24h", "wx{h}_inversion_mean_24h", "wx{h}_wind_mean_24h", "wx{h}_precip_24h_mm"]
|
| 31 |
+
CYCLIC = {"hour_local": 24, "doy_local": 366}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _is_pm(c):
|
| 35 |
+
return c.startswith(("pm25_", "region_pm25")) and c not in ("pm25_obs_24h", "pm25_change_24h", "pm25_std_24h")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class LinearForecaster:
|
| 39 |
+
def __init__(self, variant="small", alpha=ALPHA):
|
| 40 |
+
self.cfg = {"variant": variant, "alpha": alpha}
|
| 41 |
+
self.models = {}
|
| 42 |
+
self.features = {}
|
| 43 |
+
self.stations = None
|
| 44 |
+
|
| 45 |
+
def _matrix(self, df, h):
|
| 46 |
+
"""The design matrix for horizon h, in a fixed column order."""
|
| 47 |
+
if self.cfg["variant"] == "small":
|
| 48 |
+
cols = [c.format(h=h) for c in SMALL]
|
| 49 |
+
X = df[cols].astype("float64")
|
| 50 |
+
else:
|
| 51 |
+
# A horizon's model sees only its own weather block (wx24_* for t+24), like the small variant.
|
| 52 |
+
cols = [c for c in feature_columns(df)
|
| 53 |
+
if c != "station_id" and (not c.startswith("wx") or c.startswith(f"wx{h}_"))
|
| 54 |
+
and (not c.startswith("fireworks_") or c == f"fireworks_t{h}")]
|
| 55 |
+
X = df[cols].astype("float64")
|
| 56 |
+
for c, period in CYCLIC.items():
|
| 57 |
+
X[f"{c}_sin"], X[f"{c}_cos"] = np.sin(2 * np.pi * X[c] / period), np.cos(2 * np.pi * X[c] / period)
|
| 58 |
+
rad = np.deg2rad(X[f"wx{h}_wind_dir_deg"])
|
| 59 |
+
X[f"wx{h}_wind_dir_sin"], X[f"wx{h}_wind_dir_cos"] = np.sin(rad), np.cos(rad)
|
| 60 |
+
X = X.drop(columns=[*CYCLIC, f"wx{h}_wind_dir_deg"])
|
| 61 |
+
stations = pd.Categorical(df["station_id"], categories=self.stations)
|
| 62 |
+
X = pd.concat([X, pd.get_dummies(stations, prefix="station", dtype="float64").set_index(X.index)], axis=1)
|
| 63 |
+
for c in X.columns:
|
| 64 |
+
if _is_pm(c):
|
| 65 |
+
X[c] = np.log1p(X[c].clip(lower=0))
|
| 66 |
+
return X
|
| 67 |
+
|
| 68 |
+
def fit(self, train):
|
| 69 |
+
self.stations = sorted(train["station_id"].unique())
|
| 70 |
+
for h, label in zip(HORIZONS, LABELS):
|
| 71 |
+
rows = train[train[label].notna()]
|
| 72 |
+
X = self._matrix(rows, h)
|
| 73 |
+
self.features[h] = list(X.columns)
|
| 74 |
+
model = make_pipeline(SimpleImputer(strategy="median", add_indicator=True, keep_empty_features=True),
|
| 75 |
+
StandardScaler(), Ridge(alpha=self.cfg["alpha"]))
|
| 76 |
+
self.models[h] = model.fit(X.to_numpy(), np.log1p(rows[label].clip(lower=0)).to_numpy())
|
| 77 |
+
return self
|
| 78 |
+
|
| 79 |
+
def predict(self, df):
|
| 80 |
+
return pd.DataFrame({p: np.expm1(self.models[h].predict(self._matrix(df, h).to_numpy())).clip(min=0)
|
| 81 |
+
for h, p in zip(HORIZONS, PREDS)}, index=df.index)
|
| 82 |
+
|
| 83 |
+
def coefficients(self):
|
| 84 |
+
"""Standardized coefficients per horizon (log-space effect of a one-standard-deviation change)."""
|
| 85 |
+
out = {}
|
| 86 |
+
for h, model in self.models.items():
|
| 87 |
+
imputer, ridge = model.named_steps["simpleimputer"], model.named_steps["ridge"]
|
| 88 |
+
names = self.features[h] + [f"{self.features[h][i]}_missing" for i in imputer.indicator_.features_]
|
| 89 |
+
out[h] = pd.Series(ridge.coef_, index=[n.replace(f"wx{h}_", "wx_") for n in names])
|
| 90 |
+
return pd.DataFrame(out).sort_values(HORIZONS[0], key=abs, ascending=False)
|
| 91 |
+
|
| 92 |
+
def save(self, path):
|
| 93 |
+
path = Path(path)
|
| 94 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 95 |
+
for h, model in self.models.items():
|
| 96 |
+
joblib.dump(model, path / f"t{h}.joblib")
|
| 97 |
+
(path / "meta.json").write_text(json.dumps({
|
| 98 |
+
"cfg": self.cfg, "stations": self.stations, "features": {str(h): f for h, f in self.features.items()},
|
| 99 |
+
}, indent=2))
|
| 100 |
+
|
| 101 |
+
@classmethod
|
| 102 |
+
def load(cls, path):
|
| 103 |
+
path = Path(path)
|
| 104 |
+
meta = json.loads((path / "meta.json").read_text())
|
| 105 |
+
m = cls(**meta["cfg"])
|
| 106 |
+
m.stations, m.features = meta["stations"], {int(h): f for h, f in meta["features"].items()}
|
| 107 |
+
m.models = {h: joblib.load(path / f"t{h}.joblib") for h in HORIZONS}
|
| 108 |
+
return m
|
linear/code/aq_models/pyfunc.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MLflow wrapper, so every model family registers and loads the same way.
|
| 2 |
+
|
| 3 |
+
Input: `gold_features` rows (keys + features). Output: `station_id`, `hour_utc`, `pred_t24/48/72` for every
|
| 4 |
+
input row. The GRU returns NaN for rows without 72 consecutive hours of history in the input, so batch
|
| 5 |
+
inference passes it the last 72 hours per station.
|
| 6 |
+
|
| 7 |
+
The model files are logged as an artifact and this package as code (`code_paths`), so a registered version
|
| 8 |
+
loads anywhere with `mlflow.pyfunc.load_model("models:/<name>@champion")`.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import mlflow
|
| 12 |
+
import pandas as pd
|
| 13 |
+
|
| 14 |
+
from .common import KEYS
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class Forecaster(mlflow.pyfunc.PythonModel):
|
| 18 |
+
def __init__(self, kind):
|
| 19 |
+
self.kind = kind
|
| 20 |
+
|
| 21 |
+
def load_context(self, context):
|
| 22 |
+
if self.kind == "lightgbm":
|
| 23 |
+
from .lgbm import LightGBMForecaster as cls
|
| 24 |
+
elif self.kind == "linear":
|
| 25 |
+
from .linear import LinearForecaster as cls
|
| 26 |
+
else:
|
| 27 |
+
from .gru import GRUForecaster as cls
|
| 28 |
+
self.model = cls.load(context.artifacts["model"])
|
| 29 |
+
|
| 30 |
+
def predict(self, context, model_input, params=None):
|
| 31 |
+
return pd.concat([model_input[KEYS].reset_index(drop=True),
|
| 32 |
+
self.model.predict(model_input).reset_index(drop=True)], axis=1)
|
linear/conda.yaml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
channels:
|
| 2 |
+
- conda-forge
|
| 3 |
+
dependencies:
|
| 4 |
+
- python=3.12.3
|
| 5 |
+
- pip<=25.0.1
|
| 6 |
+
- pip:
|
| 7 |
+
- mlflow==3.18.0
|
| 8 |
+
- scikit-learn==1.9.1
|
| 9 |
+
- pandas
|
| 10 |
+
- numpy
|
| 11 |
+
name: mlflow-env
|
linear/metadata/MLmodel
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
artifact_path: dbfs:/databricks/mlflow-tracking/50496446126707/logged_models/m-6929babe95a54d199a1e15dce7d95fa3/artifacts
|
| 2 |
+
databricks_runtime: client.4
|
| 3 |
+
flavors:
|
| 4 |
+
python_function:
|
| 5 |
+
artifacts:
|
| 6 |
+
model:
|
| 7 |
+
path: artifacts/linear
|
| 8 |
+
uri: /tmp/tmpk5u_j3kw/linear
|
| 9 |
+
cloudpickle_version: 3.0.0
|
| 10 |
+
code: code
|
| 11 |
+
env:
|
| 12 |
+
conda: conda.yaml
|
| 13 |
+
virtualenv: python_env.yaml
|
| 14 |
+
loader_module: mlflow.pyfunc.model
|
| 15 |
+
python_model: python_model.pkl
|
| 16 |
+
python_version: 3.12.3
|
| 17 |
+
streamable: false
|
| 18 |
+
is_signature_from_type_hint: false
|
| 19 |
+
mlflow_version: 3.18.0
|
| 20 |
+
model_id: m-6929babe95a54d199a1e15dce7d95fa3
|
| 21 |
+
model_size_bytes: 6941
|
| 22 |
+
model_uuid: m-6929babe95a54d199a1e15dce7d95fa3
|
| 23 |
+
prompts: null
|
| 24 |
+
run_id: 72c567f94a134640953c4ee266d4655d
|
| 25 |
+
signature:
|
| 26 |
+
inputs: '[{"type": "string", "name": "station_id", "required": true}, {"type": "datetime",
|
| 27 |
+
"name": "hour_utc", "required": true}, {"type": "double", "name": "latitude",
|
| 28 |
+
"required": true}, {"type": "double", "name": "longitude", "required": true},
|
| 29 |
+
{"type": "boolean", "name": "wasatch_front", "required": true}, {"type": "double",
|
| 30 |
+
"name": "pm25_ugm3", "required": false}, {"type": "double", "name": "no2_ppb",
|
| 31 |
+
"required": false}, {"type": "double", "name": "pm25_lag_1h", "required": false},
|
| 32 |
+
{"type": "double", "name": "pm25_lag_2h", "required": false}, {"type": "double",
|
| 33 |
+
"name": "pm25_lag_3h", "required": false}, {"type": "double", "name": "pm25_lag_6h",
|
| 34 |
+
"required": false}, {"type": "double", "name": "pm25_lag_12h", "required": false},
|
| 35 |
+
{"type": "double", "name": "pm25_lag_24h", "required": false}, {"type": "double",
|
| 36 |
+
"name": "pm25_lag_48h", "required": true}, {"type": "double", "name": "pm25_lag_72h",
|
| 37 |
+
"required": false}, {"type": "double", "name": "pm25_mean_3h", "required": true},
|
| 38 |
+
{"type": "double", "name": "pm25_mean_6h", "required": true}, {"type": "double",
|
| 39 |
+
"name": "pm25_mean_12h", "required": true}, {"type": "double", "name": "pm25_mean_24h",
|
| 40 |
+
"required": true}, {"type": "double", "name": "pm25_mean_72h", "required": true},
|
| 41 |
+
{"type": "double", "name": "pm25_mean_168h", "required": true}, {"type": "double",
|
| 42 |
+
"name": "pm25_max_24h", "required": true}, {"type": "double", "name": "pm25_std_24h",
|
| 43 |
+
"required": true}, {"type": "long", "name": "pm25_obs_24h", "required": true},
|
| 44 |
+
{"type": "double", "name": "pm25_change_24h", "required": false}, {"type": "double",
|
| 45 |
+
"name": "no2_mean_24h", "required": true}, {"type": "double", "name": "region_pm25",
|
| 46 |
+
"required": true}, {"type": "double", "name": "region_pm25_mean_24h", "required":
|
| 47 |
+
true}, {"type": "integer", "name": "hour_local", "required": true}, {"type": "integer",
|
| 48 |
+
"name": "dow_local", "required": true}, {"type": "integer", "name": "doy_local",
|
| 49 |
+
"required": true}, {"type": "boolean", "name": "fireworks_t24", "required": true},
|
| 50 |
+
{"type": "boolean", "name": "fireworks_t48", "required": true}, {"type": "boolean",
|
| 51 |
+
"name": "fireworks_t72", "required": true}, {"type": "double", "name": "wx24_temp_c",
|
| 52 |
+
"required": false}, {"type": "double", "name": "wx24_rh_pct", "required": false},
|
| 53 |
+
{"type": "double", "name": "wx24_wind_speed_ms", "required": false}, {"type":
|
| 54 |
+
"double", "name": "wx24_wind_dir_deg", "required": false}, {"type": "double",
|
| 55 |
+
"name": "wx24_cloud_cover_pct", "required": false}, {"type": "double", "name":
|
| 56 |
+
"wx24_shortwave_wm2", "required": false}, {"type": "double", "name": "wx24_inversion_c",
|
| 57 |
+
"required": false}, {"type": "double", "name": "wx24_wind_mean_24h", "required":
|
| 58 |
+
false}, {"type": "double", "name": "wx24_wind_max_24h", "required": false}, {"type":
|
| 59 |
+
"double", "name": "wx24_precip_24h_mm", "required": false}, {"type": "double",
|
| 60 |
+
"name": "wx24_inversion_mean_24h", "required": false}, {"type": "double", "name":
|
| 61 |
+
"wx24_pressure_change_24h_hpa", "required": false}, {"type": "double", "name":
|
| 62 |
+
"wx48_temp_c", "required": false}, {"type": "double", "name": "wx48_rh_pct", "required":
|
| 63 |
+
false}, {"type": "double", "name": "wx48_wind_speed_ms", "required": false}, {"type":
|
| 64 |
+
"double", "name": "wx48_wind_dir_deg", "required": false}, {"type": "double",
|
| 65 |
+
"name": "wx48_cloud_cover_pct", "required": false}, {"type": "double", "name":
|
| 66 |
+
"wx48_shortwave_wm2", "required": false}, {"type": "double", "name": "wx48_inversion_c",
|
| 67 |
+
"required": false}, {"type": "double", "name": "wx48_wind_mean_24h", "required":
|
| 68 |
+
false}, {"type": "double", "name": "wx48_wind_max_24h", "required": false}, {"type":
|
| 69 |
+
"double", "name": "wx48_precip_24h_mm", "required": false}, {"type": "double",
|
| 70 |
+
"name": "wx48_inversion_mean_24h", "required": false}, {"type": "double", "name":
|
| 71 |
+
"wx48_pressure_change_24h_hpa", "required": false}, {"type": "double", "name":
|
| 72 |
+
"wx72_temp_c", "required": false}, {"type": "double", "name": "wx72_rh_pct", "required":
|
| 73 |
+
false}, {"type": "double", "name": "wx72_wind_speed_ms", "required": false}, {"type":
|
| 74 |
+
"double", "name": "wx72_wind_dir_deg", "required": false}, {"type": "double",
|
| 75 |
+
"name": "wx72_cloud_cover_pct", "required": false}, {"type": "double", "name":
|
| 76 |
+
"wx72_shortwave_wm2", "required": false}, {"type": "double", "name": "wx72_inversion_c",
|
| 77 |
+
"required": false}, {"type": "double", "name": "wx72_wind_mean_24h", "required":
|
| 78 |
+
false}, {"type": "double", "name": "wx72_wind_max_24h", "required": false}, {"type":
|
| 79 |
+
"double", "name": "wx72_precip_24h_mm", "required": false}, {"type": "double",
|
| 80 |
+
"name": "wx72_inversion_mean_24h", "required": false}, {"type": "double", "name":
|
| 81 |
+
"wx72_pressure_change_24h_hpa", "required": false}]'
|
| 82 |
+
outputs: '[{"type": "string", "name": "station_id", "required": true}, {"type":
|
| 83 |
+
"datetime", "name": "hour_utc", "required": true}, {"type": "double", "name":
|
| 84 |
+
"pred_t24", "required": true}, {"type": "double", "name": "pred_t48", "required":
|
| 85 |
+
true}, {"type": "double", "name": "pred_t72", "required": true}]'
|
| 86 |
+
params: null
|
| 87 |
+
type_hint_from_example: false
|
| 88 |
+
utc_time_created: '2026-11-05 14:26:51.280529'
|
linear/metadata/conda.yaml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
channels:
|
| 2 |
+
- conda-forge
|
| 3 |
+
dependencies:
|
| 4 |
+
- python=3.12.3
|
| 5 |
+
- pip<=25.0.1
|
| 6 |
+
- pip:
|
| 7 |
+
- mlflow==3.18.0
|
| 8 |
+
- scikit-learn==1.9.1
|
| 9 |
+
- pandas
|
| 10 |
+
- numpy
|
| 11 |
+
name: mlflow-env
|
linear/metadata/python_env.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
python: 3.12.3
|
| 2 |
+
build_dependencies:
|
| 3 |
+
- pip==25.0.1
|
| 4 |
+
- setuptools==74.0.0
|
| 5 |
+
- wheel==0.45.1
|
| 6 |
+
dependencies:
|
| 7 |
+
- -r requirements.txt
|
linear/metadata/requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
mlflow==3.18.0
|
| 2 |
+
scikit-learn==1.9.1
|
| 3 |
+
pandas
|
| 4 |
+
numpy
|
linear/python_env.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
python: 3.12.3
|
| 2 |
+
build_dependencies:
|
| 3 |
+
- pip==25.0.1
|
| 4 |
+
- setuptools==74.0.0
|
| 5 |
+
- wheel==0.45.1
|
| 6 |
+
dependencies:
|
| 7 |
+
- -r requirements.txt
|
linear/python_model.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a26fd0037ac82e2e706b7a96dc746bbffa84c40ad5e07123617f4cb5a25f9e6a
|
| 3 |
+
size 160
|
linear/requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
mlflow==3.18.0
|
| 2 |
+
scikit-learn==1.9.1
|
| 3 |
+
pandas
|
| 4 |
+
numpy
|