fix: only send GitHub token to github hosts; add basic logging

src/devnet/core/http.py

@@ -6,6 +6,7 @@ place, inherited by every source.
 from __future__ import annotations
 
 import ipaddress
+import logging
 import os
 import re
 import threading
@@ -17,6 +18,8 @@ import requests
 
 from .errors import AuthRequired, RateLimited, UpstreamError
 
+logger = logging.getLogger("devnet.http")
+
 UA = "devnet/0.1"
 TIMEOUT = 20
 MAX_BYTES = 4 * 1024 * 1024
@@ -24,6 +27,14 @@ MAX_BYTES = 4 * 1024 * 1024
 _BLOCKED_HOSTS = {"localhost", "instance-data", "metadata.google.internal"}
 _BLOCKED_SUFFIXES = (".local", ".localhost", ".internal", ".lan", ".home.arpa")
 
+# Only these hostnames ever receive the GitHub bearer token. Every other upstream
+# gets a clean request with no credentials attached. Before this guard existed the
+# token was attached to every outbound request - pypi, npm, crates, HN, every RSS
+# target, SO, HF. That was a live secret leak.
+_GITHUB_HOSTS = frozenset(
+    {"github.com", "www.github.com", "api.github.com", "raw.githubusercontent.com"}
+)
+
 # ponytail: in-process TTL cache. Ceiling: one container, lost on restart, nothing
 # shared between replicas. Upgrade: Redis when a second replica exists.
 _cache: dict[str, tuple[float, Any]] = {}
@@ -70,18 +81,25 @@ def strip_html(text: str | None) -> str | None:
     return " ".join(_TAG_RE.sub(" ", text).split()) or None
 
 
-def _headers(extra: dict | None = None) -> dict[str, str]:
+def _is_github(url: str) -> bool:
+    host = (urlsplit(url).hostname or "").lower()
+    return host in _GITHUB_HOSTS or host.endswith(".githubusercontent.com") or host.endswith(".github.com")
+
+
+def _headers(extra: dict | None = None, *, github: bool = False) -> dict[str, str]:
     headers = {"User-Agent": UA, "Accept": "application/json"}
-    token = os.environ.get("GITHUB_TOKEN")
-    if token:
-        headers["Authorization"] = f"Bearer {token}"
-    
+    if github:
+        token = os.environ.get("GITHUB_TOKEN")
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
     if extra:
-        headers.update(extra)
-        # إذا تم تمرير Authorization كـ None في extra، يتم حذف الترويسة لمنع تسريب التوكن
-        if extra.get("Authorization") is None and "Authorization" in headers:
-            headers.pop("Authorization")
-            
+        # None values mean "strip this header" so callers can suppress one
+        # without knowing which token a previous layer set.
+        for k, v in extra.items():
+            if v is None:
+                headers.pop(k, None)
+            else:
+                headers[k] = v
     return headers
 
 
@@ -96,22 +114,30 @@ def get(
     """Fetch JSON or text with a TTL cache and four-state error mapping."""
 
     def produce() -> Any:
+        url_clean = url.split("?", 1)[0]  # never log query strings - may carry keys
         try:
-            resp = requests.get(url, params=params, headers=_headers(headers), timeout=TIMEOUT)
+            resp = requests.get(
+                url,
+                params=params,
+                headers=_headers(headers, github=_is_github(url)),
+                timeout=TIMEOUT,
+            )
         except requests.RequestException as exc:
+            logger.warning("request failed: %s %s", type(exc).__name__, url_clean)
             raise UpstreamError(f"{url}: {exc}") from exc
+        logger.debug("%s %s", resp.status_code, url_clean)
         if resp.status_code == 429:
+            logger.info("rate limited: %s", url_clean)
             raise RateLimited(f"{url}: upstream rate limit")
         if resp.status_code in (401, 403):
             body = resp.text[:300].lower()
             if resp.headers.get("x-ratelimit-remaining") == "0" or "rate limit" in body:
+                logger.info("rate limited: %s", url_clean)
                 raise RateLimited(f"{url}: upstream rate limit")
             raise AuthRequired(f"{url}: HTTP {resp.status_code}")
         if resp.status_code >= 400:
-            # Keep the upstream's own message - SO/HF return JSON error bodies that
-            # explain why (bad key, missing filter, backoff), and the HTTP code alone
-            # hides it.
             snippet = (resp.text or "").strip().replace("\n", " ")[:200]
+            logger.warning("upstream %s: %s %s", resp.status_code, url_clean, snippet)
             raise UpstreamError(f"{url}: HTTP {resp.status_code} {snippet}")
         if len(resp.content) > MAX_BYTES:
             raise UpstreamError(f"{url}: response over {MAX_BYTES} bytes")
@@ -120,6 +146,7 @@ def get(
     key = f"{url}|{params}|{headers}|{as_json}"
     hit = _cache.get(key)
     if hit and time.time() - hit[0] < ttl:
+        logger.debug("cache hit: %s", url.split("?", 1)[0])
         return hit[1]
     value = produce()
     with _lock: