File size: 3,998 Bytes
5bfc214 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | """Disk-cached, backoff-aware HTTP GET, shared by the OOD miner and (if
needed later) other long-running API-bound collection scripts.
Every response is cached to disk keyed by a caller-supplied cache key, so a
killed-and-restarted job never re-issues a request it already got an answer
for — the cache alone makes these scripts resume-safe, no separate progress
file needed. Respects both NVD-style plain 403/429 backoff and GitHub-style
`X-RateLimit-Remaining`/`X-RateLimit-Reset` headers.
"""
from __future__ import annotations
import hashlib
import json
import time
from pathlib import Path
import requests
def _cache_path(cache_dir: Path, cache_key: str) -> Path:
h = hashlib.sha256(cache_key.encode("utf-8")).hexdigest()[:32]
return cache_dir / f"{h}.json"
def cached_get(
session: requests.Session,
url: str,
cache_dir: Path,
cache_key: str,
params: dict | None = None,
max_retries: int = 6,
base_timeout: int = 30,
) -> dict | None:
"""Returns the parsed JSON body, or None on a non-retryable failure
(e.g. 404 — a real "this doesn't exist", not a rate limit). Caches only
successful (200) responses; failures are never cached, so a transient
error gets retried on the next run rather than permanently poisoning
the cache.
Callers that apply their own rate-limit delay between calls (as
scripts/build_ood_set.py does) should check `is_cached` first and skip
the delay on a hit — otherwise resuming past thousands of already-cached
requests pays the full per-call delay for zero actual network traffic.
"""
data, _ = cached_get_ex(session, url, cache_dir, cache_key, params, max_retries, base_timeout)
return data
def is_cached(cache_dir: Path, cache_key: str) -> bool:
return _cache_path(cache_dir, cache_key).exists()
def cached_get_ex(
session: requests.Session,
url: str,
cache_dir: Path,
cache_key: str,
params: dict | None = None,
max_retries: int = 6,
base_timeout: int = 30,
) -> tuple[dict | None, bool]:
"""Same as `cached_get`, but also returns whether the result came from
the on-disk cache (True) or a live network call (False)."""
cache_dir.mkdir(parents=True, exist_ok=True)
path = _cache_path(cache_dir, cache_key)
if path.exists():
try:
return json.loads(path.read_text()), True
except (json.JSONDecodeError, OSError):
pass # corrupt cache entry — fall through and refetch
for attempt in range(max_retries):
try:
resp = session.get(url, params=params, timeout=base_timeout)
except requests.RequestException:
time.sleep(min(2**attempt, 60))
continue
if resp.status_code == 200:
try:
data = resp.json()
except ValueError:
return None, False
path.write_text(json.dumps(data))
return data, False
if resp.status_code == 404:
return None, False # real "not found" — don't retry, don't cache
if resp.status_code in (403, 429):
wait_s = _backoff_seconds(resp, attempt)
time.sleep(wait_s)
continue
if 500 <= resp.status_code < 600:
time.sleep(min(2**attempt, 60))
continue
return None, False # other 4xx: treat as non-retryable
return None, False
def _backoff_seconds(resp: requests.Response, attempt: int) -> float:
retry_after = resp.headers.get("Retry-After")
if retry_after:
try:
return float(retry_after) + 1
except ValueError:
pass
reset = resp.headers.get("X-RateLimit-Reset")
remaining = resp.headers.get("X-RateLimit-Remaining")
if reset and remaining == "0":
wait = float(reset) - time.time()
if wait > 0:
return min(wait + 1, 900) # cap at 15 min so one bad header can't wedge the job
return min(2**attempt * 2, 120)
|