source/APIs/http_client.py

"""Shared HTTP plumbing for the requests-based API wrappers (RAWG, IGDB,
Metacritic). Fixes the failure modes the original one-off wrappers shipped
with: no timeout on any call (infinite-hang risk), no retry on 429/5xx, and a
fresh TCP connection per request.

Gamespot has its own fetcher stack (curl_cffi / Playwright with Cloudflare
challenge handling) and HLTB goes through howlongtobeatpy — neither uses this.

Behavior: retries with exponential backoff on 429, 5xx and connection errors;
any other HTTP error (or exhausted retries) raises — callers (enrichment,
create_game_dataset) already catch per-source exceptions and degrade.
"""

import time

import requests

TIMEOUT = 20  # seconds, applied to every request
RETRIES = 3
BACKOFF = 2.0  # seconds; doubles per retry (2, 4, 8)
RETRY_STATUSES = {429, 500, 502, 503, 504}


class HttpClient:
    """A small requests.Session wrapper: timeout + retry/backoff + reuse."""

    def __init__(self, headers: dict | None = None):
        self.session = requests.Session()
        if headers:
            self.session.headers.update(headers)

    def request(self, method: str, url: str, **kwargs) -> requests.Response:
        kwargs.setdefault("timeout", TIMEOUT)
        last_exc = None
        for attempt in range(RETRIES):
            try:
                resp = self.session.request(method, url, **kwargs)
            except (requests.ConnectionError, requests.Timeout) as e:
                last_exc = e
                time.sleep(BACKOFF * (2 ** attempt))
                continue
            if resp.status_code in RETRY_STATUSES and attempt < RETRIES - 1:
                # Honor Retry-After when the API provides one (429s often do).
                retry_after = resp.headers.get("Retry-After")
                try:
                    delay = float(retry_after) if retry_after else BACKOFF * (2 ** attempt)
                except ValueError:
                    delay = BACKOFF * (2 ** attempt)
                time.sleep(min(delay, 60.0))
                continue
            resp.raise_for_status()
            return resp
        raise last_exc if last_exc else requests.ConnectionError(f"retries exhausted for {url}")

    def get_json(self, url: str, **kwargs) -> dict | list:
        return self.request("GET", url, **kwargs).json()

    def post(self, url: str, **kwargs) -> requests.Response:
        return self.request("POST", url, **kwargs)