Sync from GitHub (tests passed)

app/ai_engine.py

@@ -47,7 +47,12 @@ from app.settings import get_settings
 from app.features import build_feature_matrix, get_feature_descriptions
 from app.lock import pipeline_lock
 from app.async_bridge import run_async_from_sync
-from app.openrouter_client import OpenRouterError, OpenRouterRateLimitError, create_chat_completion
+from app.openrouter_client import (
+    OpenRouterError,
+    OpenRouterRateLimitError,
+    create_chat_completion,
+    get_configured_api_keys,
+)
 from pipelines.market_calendar import assign_market_date, is_after_close_news
 
 logging.basicConfig(
@@ -649,7 +654,7 @@ async def score_batch_with_llm(
     """
     settings = get_settings()
 
-    if not settings.openrouter_api_key:
+    if not get_configured_api_keys(settings):
         raise RuntimeError("OpenRouter API key not configured")
 
     normalized_articles = [
@@ -670,7 +675,7 @@ async def score_batch_with_llm(
 
     async def _request_scoring(*, max_tokens: int) -> dict[str, Any]:
         request_kwargs: dict[str, Any] = {
-            "api_key": settings.openrouter_api_key,
+            "api_keys": get_configured_api_keys(settings),
             "model": model_name,
             "messages": [
                 {"role": "system", "content": LLM_SENTIMENT_SYSTEM_PROMPT},
@@ -697,7 +702,7 @@ async def score_batch_with_llm(
             f"MALFORMED:\n{malformed_content}"
         )
         repair_data = await create_chat_completion(
-            api_key=settings.openrouter_api_key,
+            api_keys=get_configured_api_keys(settings),
             model=model_name,
             messages=[
                 {
@@ -1018,7 +1023,7 @@ async def _repair_json_response_v2(
         f"MALFORMED:\n{malformed_content}"
     )
     repair_data = await create_chat_completion(
-        api_key=settings.openrouter_api_key,
+        api_keys=get_configured_api_keys(settings),
         model=model_name,
         messages=[
             {
@@ -1086,7 +1091,7 @@ async def _score_subset_with_model_v2(
 
     async def _request(*, max_tokens: int) -> dict[str, Any]:
         request_kwargs: dict[str, Any] = {
-            "api_key": settings.openrouter_api_key,
+            "api_keys": get_configured_api_keys(settings),
             "model": model_name,
             "messages": [
                 {"role": "system", "content": LLM_V2_SYSTEM_PROMPT},
@@ -1197,7 +1202,7 @@ async def _score_batch_with_llm_v2_unbounded(
     Commodity-aware sentiment scoring with fast+reliable model escalation.
     """
     settings = get_settings()
-    if not settings.openrouter_api_key:
+    if not get_configured_api_keys(settings):
         raise RuntimeError("OpenRouter API key not configured")
 
     fast_model = settings.resolved_scoring_fast_model
@@ -1574,7 +1579,7 @@ def score_unscored_processed_articles(
         reliable_exhausted = rate_limited_by_model.get(reliable_model) == today_utc
         both_exhausted = fast_exhausted and reliable_exhausted
 
-        if settings.openrouter_api_key and llm_budget_remaining > 0 and not both_exhausted:
+        if get_configured_api_keys(settings) and llm_budget_remaining > 0 and not both_exhausted:
             eligible_items = [item for item in chunk_items if item["id"] in llm_eligible_ids]
             skipped = len(chunk_items) - len(eligible_items)
             if skipped > 0:
@@ -1587,7 +1592,7 @@ def score_unscored_processed_articles(
             llm_budget_remaining -= llm_take
             for item in eligible_items[llm_take:]:
                 fallback_reason_by_id[item["id"]] = "budget_policy"
-        elif not settings.openrouter_api_key:
+        elif not get_configured_api_keys(settings):
             for article_id in llm_eligible_ids:
                 fallback_reason_by_id[article_id] = "auth"
         elif both_exhausted:
@@ -2038,13 +2043,13 @@ def score_unscored_articles(
         llm_candidates: list[Any] = []
         non_llm_candidates: list[Any] = []
 
-        if settings.openrouter_api_key and llm_budget_remaining > 0:
+        if get_configured_api_keys(settings) and llm_budget_remaining > 0:
             llm_take = min(len(chunk), llm_budget_remaining)
             llm_candidates = chunk[:llm_take]
             non_llm_candidates = chunk[llm_take:]
         else:
             non_llm_candidates = chunk
-            if settings.openrouter_api_key and llm_budget_remaining <= 0 and not budget_exhausted_logged:
+            if get_configured_api_keys(settings) and llm_budget_remaining <= 0 and not budget_exhausted_logged:
                 logger.info(
                     "LLM budget exhausted (%s articles). Remaining chunks use soft-neutral FinBERT fallback.",
                     settings.max_llm_articles_per_run,
@@ -2096,7 +2101,7 @@ def score_unscored_articles(
         for article in non_llm_candidates:
             reason = (
                 "llm_skipped_budget"
-                if settings.openrouter_api_key and llm_budget_remaining <= 0
+                if get_configured_api_keys(settings) and llm_budget_remaining <= 0
                 else "llm_unavailable_no_api_key"
             )
             llm_results_by_id[article.id] = _neutral_llm_result(

app/commentary.py

@@ -13,7 +13,7 @@ from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import Any, Optional
 
-from .openrouter_client import OpenRouterError, create_chat_completion
+from .openrouter_client import OpenRouterError, create_chat_completion, get_configured_api_keys
 from .sentiment import (
     BEARISH,
     BULLISH,
@@ -802,7 +802,7 @@ async def _generate_commentary_and_stance(
         tft_summary=tft_summary,
     )
 
-    if not settings.openrouter_api_key:
+    if not get_configured_api_keys(settings):
         logger.warning("OpenRouter API key not configured, using template commentary fallback")
         return CommentaryGenerationResult(
             fallback_commentary, deterministic_stance, "deterministic_fallback", None, "auth", []
@@ -908,7 +908,7 @@ Write as a seasoned commodities strategist would for institutional clients—pre
 """
 
     base_request_kwargs = {
-        "api_key": settings.openrouter_api_key,
+        "api_keys": get_configured_api_keys(settings),
         "model": settings.resolved_commentary_model,
         "messages": [
             {
@@ -976,7 +976,7 @@ Write as a seasoned commodities strategist would for institutional clients—pre
             f"<malformed_output>{malformed_content}</malformed_output>"
         )
         repair_data = await create_chat_completion(
-            api_key=settings.openrouter_api_key,
+            api_keys=get_configured_api_keys(settings),
             model=next(
                 (m for m in settings.openrouter_fallback_models_list if m != settings.resolved_commentary_model),
                 settings.resolved_scoring_fast_model if hasattr(settings, "resolved_scoring_fast_model") else settings.resolved_commentary_model,

app/openrouter_client.py

@@ -54,6 +54,8 @@ def _classify_status(status: int, body: str) -> str:
     lowered = body.lower()
     if status in (401, 403):
         return "auth"
+    if status == 402:
+        return "spend_limit"
     if status == 404:
         return "model_unavailable"
     if status == 408:
@@ -106,6 +108,27 @@ def _attempt(model: str, ordinal: int, category: str, status: Optional[int], ret
     return {"model": model, "attempt": ordinal, "category": category, "status_code": status, "retried": retried}
 
 
+def get_configured_api_keys(settings: Any) -> list[str]:
+    """Return configured OpenRouter keys without exposing them to logs."""
+    configured = getattr(settings, "openrouter_api_keys", None)
+    candidates = configured if configured is not None else [
+        getattr(settings, "openrouter_api_key", None),
+        getattr(settings, "openrouter_api_key_backup", None),
+    ]
+    if isinstance(candidates, str):
+        candidates = [candidates]
+    keys: list[str] = []
+    for candidate in candidates or []:
+        normalized = str(candidate or "").strip()
+        if normalized and normalized not in keys:
+            keys.append(normalized)
+    return keys
+
+
+def _api_key_slot(index: int) -> str:
+    return "primary" if index == 0 else f"backup-{index}"
+
+
 def _extract_message_text(data: dict[str, Any]) -> str:
     choices = data.get("choices")
     if not isinstance(choices, list) or not choices:
@@ -141,7 +164,8 @@ def _candidate_payload(base_payload: dict[str, Any], candidate: str) -> dict[str
 
 async def create_chat_completion(
     *,
-    api_key: str,
+    api_key: Optional[str] = None,
+    api_keys: Optional[list[str]] = None,
     model: str,
     messages: list[dict[str, Any]],
     max_tokens: Optional[int] = None,
@@ -157,18 +181,18 @@ async def create_chat_completion(
     extra_payload: Optional[dict[str, Any]] = None,
     chain_deadline_seconds: float = 120.0,
 ) -> dict[str, Any]:
-    """Call each model explicitly so failure category and actual model are known."""
-    if not api_key:
+    """Try model fallbacks and rotate keys on quota or spend-limit responses."""
+    configured_keys: list[str] = []
+    for candidate in [api_key, *(api_keys or [])]:
+        normalized = str(candidate or "").strip()
+        if normalized and normalized not in configured_keys:
+            configured_keys.append(normalized)
+    if not configured_keys:
         raise OpenRouterError("OpenRouter API key not configured", category="auth")
     models = _ordered_models(model, fallback_models)
     if not models:
         raise OpenRouterError("No OpenRouter model configured", category="model_unavailable")
 
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
-    if referer:
-        headers["HTTP-Referer"] = referer
-    if title:
-        headers["X-Title"] = title
     base_payload: dict[str, Any] = {"messages": messages}
     if max_tokens is not None:
         base_payload["max_tokens"] = max_tokens
@@ -185,94 +209,147 @@ async def create_chat_completion(
     deadline = time.monotonic() + max(float(chain_deadline_seconds), 1.0)
     last_error: Optional[OpenRouterError] = None
     first_rate_limit_error: Optional[OpenRouterRateLimitError] = None
-    free_tier_daily_exhausted = False
+    free_tier_daily_exhausted_keys: set[int] = set()
     retryable_categories = {"rate_limit", "timeout", "network", "provider_5xx"}
+    key_rotation_categories = {"rate_limit", "spend_limit"}
 
     async with httpx.AsyncClient(timeout=timeout_seconds) as client:
         for candidate in models:
-            if free_tier_daily_exhausted and candidate.lower().endswith(":free"):
+            candidate_is_free = candidate.lower().endswith(":free")
+            if candidate_is_free and len(free_tier_daily_exhausted_keys) == len(configured_keys):
+                logger.warning("OpenRouter free-tier daily quota exhausted for all configured keys; skipping model=%s", candidate)
                 continue
-            for attempt_index in range(max(0, int(max_retries)) + 1):
-                if time.monotonic() >= deadline:
-                    raise OpenRouterError("OpenRouter model chain deadline exceeded", category="timeout", model=candidate, attempts=attempts)
-                await _throttle_request(rpm)
-                remaining_before_call = deadline - time.monotonic()
-                if remaining_before_call <= 0:
-                    raise OpenRouterError("OpenRouter model chain deadline exceeded", category="timeout", model=candidate, attempts=attempts)
-                attempt_timeout = min(float(timeout_seconds), max(remaining_before_call, 0.1))
-                payload = _candidate_payload(base_payload, candidate)
-                response: Optional[httpx.Response] = None
-                try:
-                    response = await asyncio.wait_for(
-                        client.post("https://openrouter.ai/api/v1/chat/completions", headers=headers, json=payload),
-                        timeout=attempt_timeout,
-                    )
-                except (httpx.TimeoutException, asyncio.TimeoutError, TimeoutError) as exc:
-                    last_error = OpenRouterError(str(exc) or "OpenRouter request timed out", category="timeout", model=candidate)
-                except httpx.RequestError as exc:
-                    last_error = OpenRouterError(str(exc), category="network", model=candidate)
-                else:
-                    if response.status_code == 200:
-                        _log_rate_limit_headers(response, candidate)
-                        try:
-                            data = response.json()
-                        except ValueError:
-                            last_error = OpenRouterError("OpenRouter returned a non-JSON response", status_code=200, category="empty_response", model=candidate)
-                        else:
-                            if not data.get("choices"):
-                                last_error = OpenRouterError("OpenRouter returned no completion choices", status_code=200, category="empty_response", model=candidate)
-                            elif not _extract_message_text(data):
-                                last_error = OpenRouterError("OpenRouter returned empty message content", status_code=200, category="empty_response", model=candidate)
-                            else:
-                                attempts.append(_attempt(candidate, attempt_index + 1, "success", 200, attempt_index > 0))
-                                data["_coppermind"] = {"requested_model": candidate, "actual_model": data.get("model") or candidate, "attempts": attempts}
-                                return data
+            attempted_for_model = False
+            for key_index, active_key in enumerate(configured_keys):
+                if candidate_is_free and key_index in free_tier_daily_exhausted_keys:
+                    continue
+                attempted_for_model = True
+                key_slot = _api_key_slot(key_index)
+                headers = {"Authorization": f"Bearer {active_key}", "Content-Type": "application/json"}
+                if referer:
+                    headers["HTTP-Referer"] = referer
+                if title:
+                    headers["X-Title"] = title
+
+                rotate_key = False
+                daily_quota_exhausted = False
+                for attempt_index in range(max(0, int(max_retries)) + 1):
+                    if time.monotonic() >= deadline:
+                        raise OpenRouterError("OpenRouter model chain deadline exceeded", category="timeout", model=candidate, attempts=attempts)
+                    await _throttle_request(rpm)
+                    remaining_before_call = deadline - time.monotonic()
+                    if remaining_before_call <= 0:
+                        raise OpenRouterError("OpenRouter model chain deadline exceeded", category="timeout", model=candidate, attempts=attempts)
+                    attempt_timeout = min(float(timeout_seconds), max(remaining_before_call, 0.1))
+                    payload = _candidate_payload(base_payload, candidate)
+                    response: Optional[httpx.Response] = None
+                    try:
+                        response = await asyncio.wait_for(
+                            client.post("https://openrouter.ai/api/v1/chat/completions", headers=headers, json=payload),
+                            timeout=attempt_timeout,
+                        )
+                    except (httpx.TimeoutException, asyncio.TimeoutError, TimeoutError) as exc:
+                        last_error = OpenRouterError(str(exc) or "OpenRouter request timed out", category="timeout", model=candidate)
+                    except httpx.RequestError as exc:
+                        last_error = OpenRouterError(str(exc), category="network", model=candidate)
                     else:
-                        body = response.text[:1000]
-                        category = _classify_status(response.status_code, body)
-                        error_cls = OpenRouterRateLimitError if category == "rate_limit" else OpenRouterError
-                        last_error = error_cls(f"OpenRouter {category}: HTTP {response.status_code} - {body[:500]}", status_code=response.status_code, category=category, model=candidate)
-                        if isinstance(last_error, OpenRouterRateLimitError) and first_rate_limit_error is None:
-                            first_rate_limit_error = last_error
-                        if _is_free_tier_daily_quota_exhausted(response.status_code, body, candidate):
-                            free_tier_daily_exhausted = True
-
-                status = response.status_code if response is not None else None
-                retryable = last_error is not None and last_error.category in retryable_categories
-                # When fallback models exist, advance immediately on a timeout instead of
-                # burning a second 45s window on the same unresponsive upstream model.
-                skip_same_model_timeout_retry = (
-                    last_error is not None
-                    and last_error.category == "timeout"
-                    and len(models) > 1
-                )
-                will_retry = (
-                    retryable
-                    and not skip_same_model_timeout_retry
-                    and not free_tier_daily_exhausted
-                    and attempt_index < max(0, int(max_retries))
-                )
-                attempts.append(_attempt(candidate, attempt_index + 1, last_error.category if last_error else "network", status, will_retry))
-                if last_error and last_error.category == "auth":
-                    last_error.attempts = attempts
-                    raise last_error
-                if will_retry:
-                    retry_after = _parse_retry_after_seconds(response) if response is not None else None
-                    delay = min(retry_after if retry_after is not None else (2 ** attempt_index) + random.uniform(0.0, 0.5), 30.0)
-                    remaining = max(deadline - time.monotonic(), 0.0)
-                    if delay >= remaining:
+                        if response.status_code == 200:
+                            _log_rate_limit_headers(response, candidate)
+                            try:
+                                data = response.json()
+                            except ValueError:
+                                last_error = OpenRouterError("OpenRouter returned a non-JSON response", status_code=200, category="empty_response", model=candidate)
+                            else:
+                                if not data.get("choices"):
+                                    last_error = OpenRouterError("OpenRouter returned no completion choices", status_code=200, category="empty_response", model=candidate)
+                                elif not _extract_message_text(data):
+                                    last_error = OpenRouterError("OpenRouter returned empty message content", status_code=200, category="empty_response", model=candidate)
+                                else:
+                                    attempts.append(_attempt(candidate, attempt_index + 1, "success", 200, attempt_index > 0) | {"key_slot": key_slot})
+                                    data["_coppermind"] = {
+                                        "requested_model": candidate,
+                                        "actual_model": data.get("model") or candidate,
+                                        "key_slot": key_slot,
+                                        "attempts": attempts,
+                                    }
+                                    return data
+                        else:
+                            body = response.text[:1000]
+                            category = _classify_status(response.status_code, body)
+                            error_cls = OpenRouterRateLimitError if category == "rate_limit" else OpenRouterError
+                            last_error = error_cls(f"OpenRouter {category}: HTTP {response.status_code} - {body[:500]}", status_code=response.status_code, category=category, model=candidate)
+                            if isinstance(last_error, OpenRouterRateLimitError) and first_rate_limit_error is None:
+                                first_rate_limit_error = last_error
+                            if _is_free_tier_daily_quota_exhausted(response.status_code, body, candidate):
+                                daily_quota_exhausted = True
+                                free_tier_daily_exhausted_keys.add(key_index)
+                                logger.warning(
+                                    "OpenRouter free-tier daily quota exhausted; skipping this key for remaining free models in this request key_slot=%s",
+                                    key_slot,
+                                )
+
+                    status = response.status_code if response is not None else None
+                    retryable = last_error is not None and last_error.category in retryable_categories
+                    # When fallback models exist, advance immediately on a timeout instead of
+                    # burning a second 45s window on the same unresponsive upstream model.
+                    skip_same_model_timeout_retry = (
+                        last_error is not None
+                        and last_error.category == "timeout"
+                        and len(models) > 1
+                    )
+                    will_retry = (
+                        retryable
+                        and not skip_same_model_timeout_retry
+                        and not daily_quota_exhausted
+                        and not (
+                            last_error
+                            and last_error.category == "rate_limit"
+                            and key_index < len(configured_keys) - 1
+                        )
+                        and attempt_index < max(0, int(max_retries))
+                    )
+                    attempts.append(
+                        _attempt(candidate, attempt_index + 1, last_error.category if last_error else "network", status, will_retry)
+                        | {"key_slot": key_slot}
+                    )
+                    if last_error and last_error.category == "auth":
+                        last_error.attempts = attempts
+                        raise last_error
+                    if daily_quota_exhausted:
+                        break
+                    if last_error and last_error.category in key_rotation_categories and key_index < len(configured_keys) - 1:
+                        logger.warning(
+                            "OpenRouter key quota/spend limit reached; advancing to backup key model=%s category=%s key_slot=%s",
+                            candidate,
+                            last_error.category,
+                            key_slot,
+                        )
+                        rotate_key = True
                         break
-                    await asyncio.sleep(delay)
+                    if will_retry:
+                        retry_after = _parse_retry_after_seconds(response) if response is not None else None
+                        delay = min(retry_after if retry_after is not None else (2 ** attempt_index) + random.uniform(0.0, 0.5), 30.0)
+                        remaining = max(deadline - time.monotonic(), 0.0)
+                        if delay >= remaining:
+                            break
+                        await asyncio.sleep(delay)
+                        continue
+                    break
+                if rotate_key:
                     continue
-                break
-            if free_tier_daily_exhausted:
-                logger.warning("OpenRouter daily free-tier quota exhausted on model=%s; skipping remaining :free candidates", candidate)
+
+            if not attempted_for_model and candidate_is_free:
+                logger.warning("OpenRouter daily free-tier quota exhausted for all keys; skipping model=%s", candidate)
             else:
                 logger.warning("OpenRouter model failed; advancing chain model=%s category=%s", candidate, last_error.category if last_error else "unknown")
 
     if first_rate_limit_error is not None and not isinstance(last_error, OpenRouterRateLimitError):
         last_error = first_rate_limit_error
     if last_error is None:
-        last_error = OpenRouterError("OpenRouter request unexpectedly terminated", category="network")
+        last_error = OpenRouterRateLimitError(
+            "OpenRouter free-tier daily quota exhausted for all configured API keys",
+            status_code=429,
+            category="rate_limit",
+        )
     last_error.attempts = attempts
     raise last_error

app/settings.py

@@ -89,36 +89,45 @@ class Settings(BaseSettings):
     
     # OpenRouter AI Commentary
     openrouter_api_key: Optional[str] = None
-    # Deprecated - kept for backward compatibility. Keep this on a currently
-    # listed free model so installations without an explicit env override do
-    # not silently start with a retired OpenRouter slug.
-    openrouter_model: str = "nvidia/nemotron-3-super-120b-a12b:free"
-    # Keep fast and reliable roles independent so one provider/model removal
-    # cannot collapse the entire LLM path. NVIDIA and Liquid passed the exact
-    # credentialed scoring/commentary probe on 2026-09-27; the Gemma free
-    # models were temporarily rate-limited by their shared upstream pool.
-    openrouter_model_scoring: str = "nvidia/nemotron-3-super-120b-a12b:free"
-    openrouter_model_scoring_fast: Optional[str] = "nvidia/nemotron-3-super-120b-a12b:free"
-    openrouter_model_scoring_reliable: Optional[str] = "liquid/lfm-2.5-2.6b:free"
-    openrouter_model_commentary: str = "nvidia/nemotron-3-super-120b-a12b:free"
-    # Stay below the shared free-provider request ceiling. The worker still
-    # uses bounded retries and a per-chain deadline for transient 429s.
+    # Rotated to when the primary key receives a rate-limit response.
+    openrouter_api_key_backup: Optional[str] = None
+    # Deprecated generic fallback. Keep it on a currently listed model that
+    # supports the project's structured-output contract.
+    openrouter_model: str = "google/gemma-4-31b-it:free"
+    # Use a high-usage free model for fast scoring and a separate provider for
+    # reliable scoring/commentary. All selected routes support response_format
+    # and reasoning in the current OpenRouter catalog.
+    openrouter_model_scoring: str = "nvidia/nemotron-3.5-super-vl:free"
+    openrouter_model_scoring_fast: Optional[str] = "nvidia/nemotron-3.5-super-vl:free"
+    openrouter_model_scoring_reliable: Optional[str] = "google/gemma-4-31b-it:free"
+    openrouter_model_commentary: str = "google/gemma-4-31b-it:free"
+    # Stay below OpenRouter's free-model per-minute ceiling. The account-wide
+    # daily allowance is separate, and bounded fallbacks can still consume it.
     openrouter_rpm: int = 12
     openrouter_max_retries: int = 1
     openrouter_timeout_seconds: float = 45.0
     openrouter_chain_deadline_seconds: float = 120.0
-    # Free tier: ~50 req/day per model. At chunk_size=12 a run of 60 articles
-    # costs ~5 chunks (=5–10 requests incl. escalation) which leaves headroom
-    # for multiple runs per day before hitting the ceiling. Raise cautiously.
+    # OpenRouter's free-model daily allowance is account-wide, not per model.
+    # Failed attempts and model fallbacks can also consume that allowance.
     max_llm_articles_per_run: int = 100
     # Comma-separated list of additional OpenRouter model slugs used by the
     # client as transport-level fallbacks when the primary model 429s/5xx's.
     # Example: "google/gemini-flash-1.5:free,meta-llama/llama-3.1-8b-instruct:free"
     openrouter_fallback_models: Optional[str] = (
-        "liquid/lfm-2.5-2.6b:free,nvidia/nemotron-3-super-120b-a12b:free,"
-        "nvidia/nemotron-3.5-super-vl:free,"
-        "google/gemma-4-31b-it:free,google/gemma-4-26b-a4b-it:free"
+        "google/gemma-4-31b-it:free,xingchen-agi/xing4.0-29b-a4b:free,"
+        "google/gemma-4-26b-a4b-it:free"
     )
+
+    @property
+    def openrouter_api_keys(self) -> list[str]:
+        """Configured OpenRouter keys in priority order, with duplicates removed."""
+        keys: list[str] = []
+        for candidate in (self.openrouter_api_key, self.openrouter_api_key_backup):
+            normalized = str(candidate or "").strip()
+            if normalized and normalized not in keys:
+                keys.append(normalized)
+        return keys
+
     tokenizers_parallelism: str = "false"
     
     # Twelve Data (Live Price)
@@ -135,7 +144,7 @@ class Settings(BaseSettings):
     
     # LLM Sentiment Analysis
     # Deprecated - kept for backward compatibility
-    llm_sentiment_model: str = "nvidia/nemotron-3-super-120b-a12b:free"
+    llm_sentiment_model: str = "google/gemma-4-31b-it:free"
     
     # Pipeline trigger authentication
     pipeline_trigger_secret: Optional[str] = None
@@ -256,7 +265,7 @@ class Settings(BaseSettings):
                 self.llm_sentiment_model,
                 self.openrouter_model,
             )
-            or "nvidia/nemotron-3-super-120b-a12b:free"
+            or "nvidia/nemotron-3.5-super-vl:free"
         )
 
     @property
@@ -274,7 +283,7 @@ class Settings(BaseSettings):
                 self.llm_sentiment_model,
                 self.openrouter_model_scoring,
             )
-            or "nvidia/nemotron-3-super-120b-a12b:free"
+            or "google/gemma-4-31b-it:free"
         )
 
     @property
@@ -322,6 +331,7 @@ def mask_api_key(text: str, settings: Settings = None) -> str:
     keys_to_mask = [
         settings.twelvedata_api_key,
         settings.openrouter_api_key,
+        settings.openrouter_api_key_backup,
         settings.newsapi_key,
         settings.pipeline_trigger_secret,
     ]

app/stepfun_probe.py

@@ -25,7 +25,7 @@ from app.ai_engine import (
 )
 from app.db import SessionLocal, init_db
 from app.models import NewsProcessed, NewsRaw
-from app.openrouter_client import OpenRouterError, create_chat_completion
+from app.openrouter_client import OpenRouterError, create_chat_completion, get_configured_api_keys
 from app.settings import get_settings
 
 
@@ -99,7 +99,7 @@ async def _run_strict_probe(
     started = time.perf_counter()
     try:
         data = await create_chat_completion(
-            api_key=settings.openrouter_api_key or "",
+            api_keys=get_configured_api_keys(settings),
             model=model,
             messages=[
                 {

scripts/probe_openrouter_contracts.py

@@ -19,11 +19,14 @@ from app.settings import get_settings
 
 async def run_probe(*, fast: str, reliable: str, commentary_model: str, batch_size: int = 12) -> dict:
     runtime = get_settings()
-    if not runtime.openrouter_api_key:
+    api_keys = runtime.openrouter_api_keys
+    if not api_keys:
         return {"ok": False, "error": "OpenRouter credential is not configured"}
 
     probe_settings = SimpleNamespace(
         openrouter_api_key=runtime.openrouter_api_key,
+        openrouter_api_key_backup=runtime.openrouter_api_key_backup,
+        openrouter_api_keys=api_keys,
         openrouter_max_retries=1,
         openrouter_rpm=runtime.openrouter_rpm,
         openrouter_timeout_seconds=runtime.openrouter_timeout_seconds,
@@ -106,9 +109,9 @@ async def run_probe(*, fast: str, reliable: str, commentary_model: str, batch_si
 
 def main() -> int:
     parser = argparse.ArgumentParser(description="Probe exact OpenRouter production contracts")
-    parser.add_argument("--fast", default="nvidia/nemotron-3-super-120b-a12b:free")
-    parser.add_argument("--reliable", default="liquid/lfm-2.5-2.6b:free")
-    parser.add_argument("--commentary", default="nvidia/nemotron-3-super-120b-a12b:free")
+    parser.add_argument("--fast", default="nvidia/nemotron-3.5-super-vl:free")
+    parser.add_argument("--reliable", default="google/gemma-4-31b-it:free")
+    parser.add_argument("--commentary", default="google/gemma-4-31b-it:free")
     parser.add_argument("--batch-size", type=int, default=12, help="Synthetic scoring items, up to the worker's 12-item chunk")
     args = parser.parse_args()
     result = asyncio.run(run_probe(fast=args.fast, reliable=args.reliable, commentary_model=args.commentary, batch_size=args.batch_size))