| """Disk-cached, backoff-aware HTTP GET, shared by the OOD miner and (if |
| needed later) other long-running API-bound collection scripts. |
| |
| Every response is cached to disk keyed by a caller-supplied cache key, so a |
| killed-and-restarted job never re-issues a request it already got an answer |
| for — the cache alone makes these scripts resume-safe, no separate progress |
| file needed. Respects both NVD-style plain 403/429 backoff and GitHub-style |
| `X-RateLimit-Remaining`/`X-RateLimit-Reset` headers. |
| """ |
| from __future__ import annotations |
|
|
| import hashlib |
| import json |
| import time |
| from pathlib import Path |
|
|
| import requests |
|
|
|
|
| def _cache_path(cache_dir: Path, cache_key: str) -> Path: |
| h = hashlib.sha256(cache_key.encode("utf-8")).hexdigest()[:32] |
| return cache_dir / f"{h}.json" |
|
|
|
|
| def cached_get( |
| session: requests.Session, |
| url: str, |
| cache_dir: Path, |
| cache_key: str, |
| params: dict | None = None, |
| max_retries: int = 6, |
| base_timeout: int = 30, |
| ) -> dict | None: |
| """Returns the parsed JSON body, or None on a non-retryable failure |
| (e.g. 404 — a real "this doesn't exist", not a rate limit). Caches only |
| successful (200) responses; failures are never cached, so a transient |
| error gets retried on the next run rather than permanently poisoning |
| the cache. |
| |
| Callers that apply their own rate-limit delay between calls (as |
| scripts/build_ood_set.py does) should check `is_cached` first and skip |
| the delay on a hit — otherwise resuming past thousands of already-cached |
| requests pays the full per-call delay for zero actual network traffic. |
| """ |
| data, _ = cached_get_ex(session, url, cache_dir, cache_key, params, max_retries, base_timeout) |
| return data |
|
|
|
|
| def is_cached(cache_dir: Path, cache_key: str) -> bool: |
| return _cache_path(cache_dir, cache_key).exists() |
|
|
|
|
| def cached_get_ex( |
| session: requests.Session, |
| url: str, |
| cache_dir: Path, |
| cache_key: str, |
| params: dict | None = None, |
| max_retries: int = 6, |
| base_timeout: int = 30, |
| ) -> tuple[dict | None, bool]: |
| """Same as `cached_get`, but also returns whether the result came from |
| the on-disk cache (True) or a live network call (False).""" |
| cache_dir.mkdir(parents=True, exist_ok=True) |
| path = _cache_path(cache_dir, cache_key) |
| if path.exists(): |
| try: |
| return json.loads(path.read_text()), True |
| except (json.JSONDecodeError, OSError): |
| pass |
|
|
| for attempt in range(max_retries): |
| try: |
| resp = session.get(url, params=params, timeout=base_timeout) |
| except requests.RequestException: |
| time.sleep(min(2**attempt, 60)) |
| continue |
|
|
| if resp.status_code == 200: |
| try: |
| data = resp.json() |
| except ValueError: |
| return None, False |
| path.write_text(json.dumps(data)) |
| return data, False |
|
|
| if resp.status_code == 404: |
| return None, False |
|
|
| if resp.status_code in (403, 429): |
| wait_s = _backoff_seconds(resp, attempt) |
| time.sleep(wait_s) |
| continue |
|
|
| if 500 <= resp.status_code < 600: |
| time.sleep(min(2**attempt, 60)) |
| continue |
|
|
| return None, False |
|
|
| return None, False |
|
|
|
|
| def _backoff_seconds(resp: requests.Response, attempt: int) -> float: |
| retry_after = resp.headers.get("Retry-After") |
| if retry_after: |
| try: |
| return float(retry_after) + 1 |
| except ValueError: |
| pass |
| reset = resp.headers.get("X-RateLimit-Reset") |
| remaining = resp.headers.get("X-RateLimit-Remaining") |
| if reset and remaining == "0": |
| wait = float(reset) - time.time() |
| if wait > 0: |
| return min(wait + 1, 900) |
| return min(2**attempt * 2, 120) |
|
|
