predictor / services /model /backtest.py
pengpeng9099's picture
Deploy World Cup 2026 Predictor (docker space)
1790bde verified
Raw History Blame Contribute Delete
3.19 kB
"""Hold-out backtest: train once on everything before a cutoff, evaluate on
matches after it. Much faster than walk-forward refit and still leak-free."""
from __future__ import annotations
import sys
import time
import json
import numpy as np
import pandas as pd
import penaltyblog as pb
from penaltyblog.metrics import rps_average, multiclass_brier_score, ignorance_score
import config
from wcmodel import load_training_frame
def _outcome(h: int, a: int) -> int:
return 0 if h > a else (1 if h == a else 2)
def holdout(df: pd.DataFrame, xi: float, cutoff: str) -> dict:
"""Train strictly before cutoff, predict after cutoff. Zero leakage."""
train = df[df["date"] < cutoff].copy()
test = df[df["date"] >= cutoff].copy()
if len(train) < 2000 or len(test) < 50:
return {"xi": xi, "n": 0}
weights = pb.models.dixon_coles_weights(
train["date"], xi=xi, base_date=train["date"].max())
model = pb.models.DixonColesGoalModel(
train["home_score"].astype(int).to_numpy(),
train["away_score"].astype(int).to_numpy(),
train["home_team"].to_numpy(), train["away_team"].to_numpy(),
weights=weights, neutral_venue=train["neutral"].astype(int).to_numpy())
model.fit()
known = set(train["home_team"]) | set(train["away_team"])
probs, outcomes = [], []
for m in test.itertuples(index=False):
if m.home_team not in known or m.away_team not in known:
continue
g = model.predict(m.home_team, m.away_team,
max_goals=config.MAX_GOALS, neutral_venue=bool(m.neutral))
probs.append([float(x) for x in g.home_draw_away])
outcomes.append(_outcome(int(m.home_score), int(m.away_score)))
probs = np.array(probs)
outcomes = np.array(outcomes)
return {
"xi": xi, "n": len(outcomes),
"rps": round(float(rps_average(probs, outcomes)), 5),
"brier": round(float(multiclass_brier_score(probs, outcomes)), 5),
"logloss": round(float(ignorance_score(probs, outcomes)), 5),
}
def tune(cutoff: str = "2024-01-01") -> dict:
df = load_training_frame()
grid = [0.0008, 0.0014, 0.0020, 0.0026, 0.0035]
results = []
for xi in grid:
t0 = time.time()
r = holdout(df, xi, cutoff)
dt = time.time() - t0
results.append(r)
print(f" xi={xi:.4f} n={r['n']} RPS={r.get('rps','?')} "
f"Brier={r.get('brier','?')} LogLoss={r.get('logloss','?')} ({dt:.0f}s)",
flush=True)
best = min(results, key=lambda r: r.get("rps", 99))
print(f"\nBEST xi={best['xi']} RPS={best['rps']}")
(config.ARTIFACTS / "backtest.json").write_text(
json.dumps({"results": results, "best": best}, indent=2), encoding="utf-8")
# retrain with best xi on all data and save
from wcmodel import WorldCupModel
best_model = WorldCupModel(xi=best["xi"]).fit(df)
best_model.save()
print(f"Model saved with xi={best['xi']}")
return {"results": results, "best": best}
if __name__ == "__main__":
c = sys.argv[1] if len(sys.argv) > 1 else "2024-01-01"
print(f"Hold-out backtest: train < {c}, eval >= {c}\n")
tune(c)