Spaces:
Sleeping
Sleeping
File size: 2,318 Bytes
ed25d75 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """Walk-forward check: how good were next-day forecasts over recent sessions?
The model is fitted once on data before the test window. Then, one day at a
time, it forecasts the next close and only afterwards sees the real one. The
parameters stay frozen, so no test data leaks into the fit.
The yardstick is the naive forecast "tomorrow closes where today closed".
Daily prices move little, so a low MAPE on its own says almost nothing. The
question is whether the model beats that.
"""
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
import pandas as pd
from .model import FIT_WINDOW, fit
TEST_DAYS = 20
@dataclass(frozen=True)
class Backtest:
dates: pd.DatetimeIndex
actual: np.ndarray
model: np.ndarray
naive: np.ndarray
model_mape: float
naive_mape: float
model_mae: float
naive_mae: float
direction_hit: float # share of days where the model got up/down right
direction_days: int
def _mape(actual: np.ndarray, pred: np.ndarray) -> float:
return float(np.mean(np.abs((actual - pred) / actual)))
def run(closes: pd.Series, test_days: int = TEST_DAYS) -> Backtest:
series = closes.iloc[-(FIT_WINDOW + test_days):]
values = series.to_numpy(dtype=float)
if len(values) < test_days + 60:
raise ValueError("not enough history for a backtest")
train, test = values[:-test_days], values[-test_days:]
result = fit(train)
preds = np.empty(test_days)
for i, actual in enumerate(test):
preds[i] = float(np.asarray(result.forecast(steps=1))[0])
result = result.append([actual]) # reveal the real close, keep params
naive = np.concatenate([[train[-1]], test[:-1]])
moved = np.sign(test - naive)
called = np.sign(preds - naive)
mask = moved != 0 # flat days have no direction to call
hit = float(np.mean(moved[mask] == called[mask])) if mask.any() else float("nan")
return Backtest(
dates=series.index[-test_days:],
actual=test,
model=preds,
naive=naive,
model_mape=_mape(test, preds),
naive_mape=_mape(test, naive),
model_mae=float(np.mean(np.abs(test - preds))),
naive_mae=float(np.mean(np.abs(test - naive))),
direction_hit=hit,
direction_days=int(mask.sum()),
)
|