odds_source.py

"""Market odds -> implied per-team expected goals, from two free feeds.

Neither feed alone is enough:

  football-data.co.uk   Pinnacle CLOSING odds (1X2, over/under 2.5, Asian
                        handicap) for every completed match, ~20 seasons back,
                        plain CSV with no key. Sharp and free, but published
                        after each round - it never covers upcoming fixtures.

  the-odds-api.com      Live prices for UPCOMING fixtures. Free tier is 500
                        credits/month and cost is markets x regions, so one
                        `h2h,totals` call for `soccer_epl` in one region costs
                        2 credits and returns every upcoming match. Refreshing
                        twice a week runs at roughly 3% of the free quota.
                        Needs ODDS_API_KEY in .env; absent, we degrade to
                        football-data only.

Both are reduced to the same shape - (round, home, away, implied home goals,
implied away goals) - so market_ratings.py can pool played and upcoming
matches into a single fit. Pooling matters most at the start of a season,
when played matches alone cannot identify twenty teams' ratings.

Prices become expected goals by removing the bookmaker's margin and inverting
an independent-Poisson scoreline model: find the goal total and home/away
split whose implied P(home win) and P(over 2.5) match the de-vigged market.

Usage:  python odds_source.py [--season 2026-2027] [--no-live]
"""
import argparse
import os
from pathlib import Path

import numpy as np
import pandas as pd
import requests
from dotenv import load_dotenv
from scipy.optimize import brentq
from scipy.stats import poisson

from build_team_totals import canonical

load_dotenv()

FOOTBALL_DATA_URL = "https://www.football-data.co.uk/mmz4281/{code}/E0.csv"
ODDS_API_URL = "https://api.the-odds-api.com/v4/sports/soccer_epl/odds"
ODDS_API_KEY = os.environ.get("ODDS_API_KEY", "").strip()

# Pinnacle closing columns. The "C" is closing - the last price before kickoff,
# which is the sharpest number the market produces. Opening odds are also in
# the file (PSH/PSD/PSA) and are materially worse; do not substitute them.
PIN_1X2 = ["PSCH", "PSCD", "PSCA"]
PIN_OU = ["PC>2.5", "PC<2.5"]
# Bet365 closing, used only when Pinnacle is missing for a match.
B365_1X2 = ["B365CH", "B365CD", "B365CA"]
B365_OU = ["B365C>2.5", "B365C<2.5"]

MATCHES_PER_ROUND = 10

# football-data.co.uk uses short names; fotmob (via canonical()) uses long ones.
FD_NAMES = {
    "Man City": "Manchester City",
    "Man United": "Manchester United",
    "Newcastle": "Newcastle United",
    "Tottenham": "Tottenham Hotspur",
    "Wolves": "Wolverhampton Wanderers",
    "West Ham": "West Ham United",
    "Brighton": "Brighton and Hove Albion",
    "Bournemouth": "AFC Bournemouth",
    "Nott'm Forest": "Nottingham Forest",
    "Leicester": "Leicester City",
    "Leeds": "Leeds United",
    "Norwich": "Norwich City",
    "Luton": "Luton Town",
    "Ipswich": "Ipswich Town",
    "Sheffield United": "Sheffield United",
    "Coventry": "Coventry City",
    "Hull": "Hull City",
    "Sunderland": "Sunderland",
}

# Poisson support. 14 is far beyond any realistic scoreline; the tail is ~0.
_K = 14
_IDX = np.arange(_K)


def team_name(raw):
    return canonical(FD_NAMES.get(str(raw).strip(), str(raw).strip()))


def season_code(season):
    """'2026-2027' -> '2627', football-data's directory code."""
    start, end = season.split("-")
    return start[2:] + end[2:]


def devig(odds):
    """Decimal odds -> probabilities with the bookmaker margin removed.

    Proportional (multiplicative) normalisation. Shin and power methods are
    marginally better on longshots, but Pinnacle's margin is thin enough that
    the difference is far below the noise in what we do downstream.
    """
    p = 1.0 / np.asarray(odds, dtype=float)
    return p / p.sum()


def _scoreline_probs(lh, la):
    """(P(home win), P(over 2.5)) under independent Poisson."""
    grid = poisson.pmf(_IDX, lh)[:, None] * poisson.pmf(_IDX, la)[None, :]
    totals = _IDX[:, None] + _IDX[None, :]
    return grid[np.tril_indices(_K, -1)].sum(), grid[totals > 2.5].sum()


def implied_goals(p_home, p_away, p_over, iters=6):
    """Expected goals per side matching the de-vigged home-win and over prices.

    Two equations, two unknowns, solved by alternating: given a split, solve
    the total against the over/under; given a total, solve the split against
    the home-win price. Converges within a few passes.

    Returns (home_goals, away_goals), or None if the prices are not solvable
    (which happens on badly mispriced or stale rows).
    """
    share = p_home / (p_home + p_away)
    total = 2.6
    for _ in range(iters):
        try:
            total = brentq(
                lambda t: _scoreline_probs(t * share, t * (1 - share))[1] - p_over,
                0.3, 9.0)
            share = brentq(
                lambda s: _scoreline_probs(total * s, total * (1 - s))[0] - p_home,
                0.02, 0.98)
        except ValueError:
            return None
    return total * share, total * (1 - share)


def _row_to_goals(row):
    """One football-data row -> implied goals, preferring Pinnacle."""
    for cols_1x2, cols_ou in ((PIN_1X2, PIN_OU), (B365_1X2, B365_OU)):
        if not all(c in row.index for c in cols_1x2 + cols_ou):
            continue
        vals = [row[c] for c in cols_1x2 + cols_ou]
        if any(pd.isna(v) or float(v) <= 1.0 for v in vals):
            continue
        ph, _, pa = devig([row[c] for c in cols_1x2])
        p_over = devig([row[c] for c in cols_ou])[0]
        return implied_goals(ph, pa, p_over)
    return None


def fetch_football_data(season, timeout=40):
    """Completed matches with closing odds. [(round, home, away, gh, ga), ...]"""
    url = FOOTBALL_DATA_URL.format(code=season_code(season))
    r = requests.get(url, timeout=timeout)
    r.raise_for_status()

    tmp = Path(os.environ.get("TEMP", ".")) / f"_fd_{season_code(season)}.csv"
    tmp.write_bytes(r.content)
    df = pd.read_csv(tmp, encoding="latin-1").dropna(how="all")
    tmp.unlink(missing_ok=True)

    out, skipped = [], 0
    for i, row in df.reset_index(drop=True).iterrows():
        goals = _row_to_goals(row)
        if goals is None:
            skipped += 1
            continue
        # football-data lists chronologically; rounds are not a column, so
        # infer them. Only used to order matches, never as a real GW label.
        out.append((i // MATCHES_PER_ROUND + 1,
                    team_name(row["HomeTeam"]), team_name(row["AwayTeam"]),
                    goals[0], goals[1]))
    print(f"[football-data] {len(out)} played matches priced"
          + (f", {skipped} skipped (no usable closing odds)" if skipped else ""))
    return out


def fetch_odds_api(regions="uk", timeout=30):
    """Upcoming fixtures from the-odds-api. [] when no key or nothing priced."""
    if not ODDS_API_KEY:
        print("[odds-api] ODDS_API_KEY not set - upcoming fixtures skipped")
        return []
    params = {"apiKey": ODDS_API_KEY, "regions": regions,
              "markets": "h2h,totals", "oddsFormat": "decimal"}
    r = requests.get(ODDS_API_URL, params=params, timeout=timeout)
    if r.status_code != 200:
        print(f"[odds-api] HTTP {r.status_code}: {r.text[:200]}")
        return []
    print(f"[odds-api] quota used {r.headers.get('x-requests-used','?')}"
          f", remaining {r.headers.get('x-requests-remaining','?')}")

    out = []
    for ev in r.json():
        home, away = ev.get("home_team"), ev.get("away_team")
        best = _best_book(ev, home, away)
        if not best:
            continue
        ph, pa, p_over = best
        goals = implied_goals(ph, pa, p_over)
        if goals:
            out.append((None, team_name(home), team_name(away), goals[0], goals[1]))
    print(f"[odds-api] {len(out)} upcoming matches priced")
    return out


def _best_book(event, home, away):
    """First bookmaker in the event quoting both h2h and a 2.5 total."""
    for bk in event.get("bookmakers", []):
        h2h = tot = None
        for m in bk.get("markets", []):
            if m["key"] == "h2h":
                h2h = m["outcomes"]
            elif m["key"] == "totals":
                tot = [o for o in m["outcomes"] if abs(o.get("point", 0) - 2.5) < 1e-6]
        if not h2h or not tot or len(tot) < 2:
            continue
        try:
            price = {o["name"]: float(o["price"]) for o in h2h}
            ph, _, pa = devig([price[home], price["Draw"], price[away]])
            over = next(o for o in tot if o["name"].lower() == "over")
            under = next(o for o in tot if o["name"].lower() == "under")
            p_over = devig([float(over["price"]), float(under["price"])])[0]
            return ph, pa, p_over
        except (KeyError, StopIteration, ValueError):
            continue
    return None


def collect(season, live=True):
    """Played + upcoming matches, as one list, for the rating fit."""
    played = fetch_football_data(season)
    upcoming = fetch_odds_api() if live else []
    return played, upcoming


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--season", default="2026-2027")
    ap.add_argument("--no-live", action="store_true",
                    help="skip the-odds-api call (saves credits)")
    args = ap.parse_args()

    played, upcoming = collect(args.season, live=not args.no_live)
    allm = played + upcoming
    if not allm:
        print("no priced matches found")
        return
    gf = [g for _, _, _, g, _ in allm] + [g for _, _, _, _, g in allm]
    print(f"\ntotal {len(allm)} matches | implied goals/team "
          f"mean {np.mean(gf):.3f}  min {min(gf):.2f}  max {max(gf):.2f}")
    for rnd, h, a, gh, ga in allm[:5]:
        tag = f"R{rnd}" if rnd else "next"
        print(f"  {tag:>5}  {h:<24} {gh:.2f} - {ga:.2f}  {a}")


if __name__ == "__main__":
    main()