| """ |
| Temporal train / validation / test split (70 / 15 / 15). |
| |
| Split is time-ordered, not random, to prevent data leakage across time. |
| Test set = most recent 15 % of rows (by timestamp). |
| Val set = the 15 % immediately before the test set. |
| Train set = everything earlier. |
| |
| Usage: |
| python -m ml.splits |
| """ |
|
|
| from __future__ import annotations |
|
|
| from pathlib import Path |
|
|
| import pandas as pd |
|
|
| PROCESSED = Path("ml/data/processed") |
|
|
|
|
| def split(src: Path | None = None) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: |
| """Load historical.parquet and return (train, val, test) DataFrames. |
| |
| No row appears in more than one split. The split boundary is computed |
| globally across all spots so that the test set always covers the most |
| recent calendar period (avoids per-spot contamination). |
| """ |
| if src is None: |
| src = PROCESSED / "historical.parquet" |
|
|
| df = pd.read_parquet(src).sort_values("timestamp").reset_index(drop=True) |
| n = len(df) |
|
|
| test_start = int(n * 0.85) |
| val_start = int(n * 0.70) |
|
|
| train = df.iloc[:val_start].copy() |
| val = df.iloc[val_start:test_start].copy() |
| test = df.iloc[test_start:].copy() |
|
|
| return train, val, test |
|
|
|
|
| def save_splits(src: Path | None = None, dst: Path | None = None) -> None: |
| """Write train / val / test parquet files to dst directory.""" |
| if dst is None: |
| dst = PROCESSED |
| dst.mkdir(parents=True, exist_ok=True) |
|
|
| train, val, test = split(src) |
|
|
| for name, df in [("train", train), ("val", val), ("test", test)]: |
| path = dst / f"{name}.parquet" |
| df.to_parquet(path, index=False) |
| earliest = df["timestamp"].min() |
| latest = df["timestamp"].max() |
| print(f" {name}: {len(df):>7,} rows {earliest.date()} → {latest.date()} → {path}") |
|
|
|
|
| if __name__ == "__main__": |
| print("Splitting historical.parquet …") |
| save_splits() |
|
|
