File size: 3,193 Bytes
1790bde
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
"""Hold-out backtest: train once on everything before a cutoff, evaluate on
matches after it. Much faster than walk-forward refit and still leak-free."""
from __future__ import annotations
import sys
import time
import json

import numpy as np
import pandas as pd
import penaltyblog as pb
from penaltyblog.metrics import rps_average, multiclass_brier_score, ignorance_score

import config
from wcmodel import load_training_frame


def _outcome(h: int, a: int) -> int:
    return 0 if h > a else (1 if h == a else 2)


def holdout(df: pd.DataFrame, xi: float, cutoff: str) -> dict:
    """Train strictly before cutoff, predict after cutoff. Zero leakage."""
    train = df[df["date"] < cutoff].copy()
    test = df[df["date"] >= cutoff].copy()
    if len(train) < 2000 or len(test) < 50:
        return {"xi": xi, "n": 0}

    weights = pb.models.dixon_coles_weights(
        train["date"], xi=xi, base_date=train["date"].max())
    model = pb.models.DixonColesGoalModel(
        train["home_score"].astype(int).to_numpy(),
        train["away_score"].astype(int).to_numpy(),
        train["home_team"].to_numpy(), train["away_team"].to_numpy(),
        weights=weights, neutral_venue=train["neutral"].astype(int).to_numpy())
    model.fit()
    known = set(train["home_team"]) | set(train["away_team"])

    probs, outcomes = [], []
    for m in test.itertuples(index=False):
        if m.home_team not in known or m.away_team not in known:
            continue
        g = model.predict(m.home_team, m.away_team,
                          max_goals=config.MAX_GOALS, neutral_venue=bool(m.neutral))
        probs.append([float(x) for x in g.home_draw_away])
        outcomes.append(_outcome(int(m.home_score), int(m.away_score)))

    probs = np.array(probs)
    outcomes = np.array(outcomes)
    return {
        "xi": xi, "n": len(outcomes),
        "rps": round(float(rps_average(probs, outcomes)), 5),
        "brier": round(float(multiclass_brier_score(probs, outcomes)), 5),
        "logloss": round(float(ignorance_score(probs, outcomes)), 5),
    }


def tune(cutoff: str = "2024-01-01") -> dict:
    df = load_training_frame()
    grid = [0.0008, 0.0014, 0.0020, 0.0026, 0.0035]
    results = []
    for xi in grid:
        t0 = time.time()
        r = holdout(df, xi, cutoff)
        dt = time.time() - t0
        results.append(r)
        print(f"  xi={xi:.4f}  n={r['n']}  RPS={r.get('rps','?')}  "
              f"Brier={r.get('brier','?')}  LogLoss={r.get('logloss','?')}  ({dt:.0f}s)",
              flush=True)
    best = min(results, key=lambda r: r.get("rps", 99))
    print(f"\nBEST xi={best['xi']}  RPS={best['rps']}")

    (config.ARTIFACTS / "backtest.json").write_text(
        json.dumps({"results": results, "best": best}, indent=2), encoding="utf-8")

    # retrain with best xi on all data and save
    from wcmodel import WorldCupModel
    best_model = WorldCupModel(xi=best["xi"]).fit(df)
    best_model.save()
    print(f"Model saved with xi={best['xi']}")
    return {"results": results, "best": best}


if __name__ == "__main__":
    c = sys.argv[1] if len(sys.argv) > 1 else "2024-01-01"
    print(f"Hold-out backtest: train < {c}, eval >= {c}\n")
    tune(c)