Spaces:
Running
Running
Download services/model/backtest.py from 2026worldcup/predictor: direct link, hf CLI and curl.
- Browser
- Download file 3.19 kB
-
https://huggingface.co/spaces/2026worldcup/predictor/resolve/main/services/model/backtest.py
- Command line
-
hf download hf://spaces/2026worldcup/predictor/services/model/backtest.py
-
curl -L -o backtest.py https://huggingface.co/spaces/2026worldcup/predictor/resolve/main/services/model/backtest.py
3.19 kB
| """Hold-out backtest: train once on everything before a cutoff, evaluate on | |
| matches after it. Much faster than walk-forward refit and still leak-free.""" | |
| from __future__ import annotations | |
| import sys | |
| import time | |
| import json | |
| import numpy as np | |
| import pandas as pd | |
| import penaltyblog as pb | |
| from penaltyblog.metrics import rps_average, multiclass_brier_score, ignorance_score | |
| import config | |
| from wcmodel import load_training_frame | |
| def _outcome(h: int, a: int) -> int: | |
| return 0 if h > a else (1 if h == a else 2) | |
| def holdout(df: pd.DataFrame, xi: float, cutoff: str) -> dict: | |
| """Train strictly before cutoff, predict after cutoff. Zero leakage.""" | |
| train = df[df["date"] < cutoff].copy() | |
| test = df[df["date"] >= cutoff].copy() | |
| if len(train) < 2000 or len(test) < 50: | |
| return {"xi": xi, "n": 0} | |
| weights = pb.models.dixon_coles_weights( | |
| train["date"], xi=xi, base_date=train["date"].max()) | |
| model = pb.models.DixonColesGoalModel( | |
| train["home_score"].astype(int).to_numpy(), | |
| train["away_score"].astype(int).to_numpy(), | |
| train["home_team"].to_numpy(), train["away_team"].to_numpy(), | |
| weights=weights, neutral_venue=train["neutral"].astype(int).to_numpy()) | |
| model.fit() | |
| known = set(train["home_team"]) | set(train["away_team"]) | |
| probs, outcomes = [], [] | |
| for m in test.itertuples(index=False): | |
| if m.home_team not in known or m.away_team not in known: | |
| continue | |
| g = model.predict(m.home_team, m.away_team, | |
| max_goals=config.MAX_GOALS, neutral_venue=bool(m.neutral)) | |
| probs.append([float(x) for x in g.home_draw_away]) | |
| outcomes.append(_outcome(int(m.home_score), int(m.away_score))) | |
| probs = np.array(probs) | |
| outcomes = np.array(outcomes) | |
| return { | |
| "xi": xi, "n": len(outcomes), | |
| "rps": round(float(rps_average(probs, outcomes)), 5), | |
| "brier": round(float(multiclass_brier_score(probs, outcomes)), 5), | |
| "logloss": round(float(ignorance_score(probs, outcomes)), 5), | |
| } | |
| def tune(cutoff: str = "2024-01-01") -> dict: | |
| df = load_training_frame() | |
| grid = [0.0008, 0.0014, 0.0020, 0.0026, 0.0035] | |
| results = [] | |
| for xi in grid: | |
| t0 = time.time() | |
| r = holdout(df, xi, cutoff) | |
| dt = time.time() - t0 | |
| results.append(r) | |
| print(f" xi={xi:.4f} n={r['n']} RPS={r.get('rps','?')} " | |
| f"Brier={r.get('brier','?')} LogLoss={r.get('logloss','?')} ({dt:.0f}s)", | |
| flush=True) | |
| best = min(results, key=lambda r: r.get("rps", 99)) | |
| print(f"\nBEST xi={best['xi']} RPS={best['rps']}") | |
| (config.ARTIFACTS / "backtest.json").write_text( | |
| json.dumps({"results": results, "best": best}, indent=2), encoding="utf-8") | |
| # retrain with best xi on all data and save | |
| from wcmodel import WorldCupModel | |
| best_model = WorldCupModel(xi=best["xi"]).fit(df) | |
| best_model.save() | |
| print(f"Model saved with xi={best['xi']}") | |
| return {"results": results, "best": best} | |
| if __name__ == "__main__": | |
| c = sys.argv[1] if len(sys.argv) > 1 else "2024-01-01" | |
| print(f"Hold-out backtest: train < {c}, eval >= {c}\n") | |
| tune(c) | |