"""Hold-out backtest: train once on everything before a cutoff, evaluate on matches after it. Much faster than walk-forward refit and still leak-free.""" from __future__ import annotations import sys import time import json import numpy as np import pandas as pd import penaltyblog as pb from penaltyblog.metrics import rps_average, multiclass_brier_score, ignorance_score import config from wcmodel import load_training_frame def _outcome(h: int, a: int) -> int: return 0 if h > a else (1 if h == a else 2) def holdout(df: pd.DataFrame, xi: float, cutoff: str) -> dict: """Train strictly before cutoff, predict after cutoff. Zero leakage.""" train = df[df["date"] < cutoff].copy() test = df[df["date"] >= cutoff].copy() if len(train) < 2000 or len(test) < 50: return {"xi": xi, "n": 0} weights = pb.models.dixon_coles_weights( train["date"], xi=xi, base_date=train["date"].max()) model = pb.models.DixonColesGoalModel( train["home_score"].astype(int).to_numpy(), train["away_score"].astype(int).to_numpy(), train["home_team"].to_numpy(), train["away_team"].to_numpy(), weights=weights, neutral_venue=train["neutral"].astype(int).to_numpy()) model.fit() known = set(train["home_team"]) | set(train["away_team"]) probs, outcomes = [], [] for m in test.itertuples(index=False): if m.home_team not in known or m.away_team not in known: continue g = model.predict(m.home_team, m.away_team, max_goals=config.MAX_GOALS, neutral_venue=bool(m.neutral)) probs.append([float(x) for x in g.home_draw_away]) outcomes.append(_outcome(int(m.home_score), int(m.away_score))) probs = np.array(probs) outcomes = np.array(outcomes) return { "xi": xi, "n": len(outcomes), "rps": round(float(rps_average(probs, outcomes)), 5), "brier": round(float(multiclass_brier_score(probs, outcomes)), 5), "logloss": round(float(ignorance_score(probs, outcomes)), 5), } def tune(cutoff: str = "2024-01-01") -> dict: df = load_training_frame() grid = [0.0008, 0.0014, 0.0020, 0.0026, 0.0035] results = [] for xi in grid: t0 = time.time() r = holdout(df, xi, cutoff) dt = time.time() - t0 results.append(r) print(f" xi={xi:.4f} n={r['n']} RPS={r.get('rps','?')} " f"Brier={r.get('brier','?')} LogLoss={r.get('logloss','?')} ({dt:.0f}s)", flush=True) best = min(results, key=lambda r: r.get("rps", 99)) print(f"\nBEST xi={best['xi']} RPS={best['rps']}") (config.ARTIFACTS / "backtest.json").write_text( json.dumps({"results": results, "best": best}, indent=2), encoding="utf-8") # retrain with best xi on all data and save from wcmodel import WorldCupModel best_model = WorldCupModel(xi=best["xi"]).fit(df) best_model.save() print(f"Model saved with xi={best['xi']}") return {"results": results, "best": best} if __name__ == "__main__": c = sys.argv[1] if len(sys.argv) > 1 else "2024-01-01" print(f"Hold-out backtest: train < {c}, eval >= {c}\n") tune(c)