"""Map an OpenRouter model id onto a source ModelConfig. The HTTP body is still built by ``OpenAICompatibleClient``. When the id matches one published card, generation settings are copied from that card. The ``model`` string stays the OpenRouter id, because that is the id OpenRouter routes. """ from __future__ import annotations import asyncio import re from dataclasses import dataclass, field from elbench.providers.openai_compatible import OpenAICompatibleClient from elbench.schemas.config import ModelConfig, ProjectConfig, RateLimitConfig, RetryConfig from elbench.schemas.evaluation import GenerationRequest OPENROUTER_API_BASE = "https://openrouter.ai/api/v1" VISITOR_KEY_ENV = "ELBENCH_VISITOR_OPENROUTER_API_KEY" # Routing variants are not rows in GET /api/v1/models. :free and :batch are. ROUTING_VARIANT_SUFFIXES = (":nitro", ":floor") DEFAULT_SYSTEM_ONE_MODELS: tuple[tuple[str, str, float | None], ...] = ( ("LiquidAI: d1-ultra", "liquid/d1-ultra", 0.042e-6), ("LiquidAI: d1", "liquid/d1", 0.020e-6), ("TypeSafe: Jev 1.14", "typesafe/jev-1.14", 0.042e-6), ("TypeSafe: Jev Latest", "~typesafe/jev-latest", 0.042e-6), ("OpenAI: GPT-6 Luna Decisions", "openai/gpt-6-luna-decisions", 0.100e-6), ("Perplexity: Decider V1.2", "perplexity/pplx-decider-v1.2", 0.020e-6), ("Perplexity: Decider Flash V1.2", "perplexity/pplx-decider-flash-v1.2", 0.010e-6), ("Microsoft: Microsoft-Decision-1", "microsoft/microsoft-decision-1", 0.042e-6), ("Cloudflare: Clef", "cloudflare/clef", 0.042e-6), ("Cloudflare: Clef Flash", "cloudflare/clef-flash", 0.019e-6), ("Cloudflare: Clef Omni", "cloudflare/clef-omni", 0.150e-6), ("Inception: Mercury Decide", "inception/mercury-decide", 0.020e-6), ("Inception: Mercury Decide (free)", "inception/mercury-decide:free", 0.0), ("Upstage: Solar Decide", "upstage/solar-decide", 0.050e-6), ("Nace.AI: Drex Max", "nace-ai/drex-max", 0.060e-6), ("ScaleDown: Classify-1", "scaledown/classify-1", 0.040e-6), ("Respan: Span-02", "respan/span-02", 0.010e-6), ("Baseten: Mammoth 1 (free)", "baseten/mammoth-1:free", 0.0), ) # Modal generation settings among the 13 published openai-compatible cards that # have empty provider_kwargs and max_tokens 8192. Temperature 0 is unanimous # across every non-mock card. The other fields are the mode of that group, not # a value the source defines for unlisted models. UNLISTED_TEMPERATURE = 0.0 UNLISTED_MAX_TOKENS = 8192 UNLISTED_TIMEOUT = 240.0 UNLISTED_STREAM = False UNLISTED_RETRY = RetryConfig( max_attempts=2, initial_delay_seconds=1.0, max_delay_seconds=20.0, backoff_multiplier=2.0, jitter_ratio=0.2, retryable_status_codes=[], ) UNLISTED_RATE_LIMITS = RateLimitConfig(max_concurrency=100) @dataclass(slots=True) class Caveat: level: str code: str message: str @dataclass(slots=True) class VisitorProtocol: openrouter_model: str model_config: ModelConfig matched_card_id: str | None matched_card_model_name: str | None responses_api: bool card_would_use_responses_api: bool | None request_branch_differences: list[str] = field(default_factory=list) caveats: list[Caveat] = field(default_factory=list) def catalog_slug(openrouter_model: str) -> str: """Slug whose card and catalog price apply. ``:nitro`` and ``:floor`` only change provider sort. ``:free`` and ``:batch`` are separate catalog rows and stay intact. """ model = openrouter_model.strip() for suffix in ROUTING_VARIANT_SUFFIXES: if model.endswith(suffix): return model[: -len(suffix)] return model def safe_model_id(openrouter_model: str) -> str: """Filesystem-safe id. The basic-education runtime uses model_id as a path segment.""" cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", openrouter_model).strip("._-") if not cleaned: cleaned = "model" return f"openrouter_{cleaned[:80]}" def match_model_card(models: dict[str, ModelConfig], openrouter_model: str) -> ModelConfig | None: published = [model for model in models.values() if model.provider_name != "mock"] exact = [model for model in published if model.model_name == openrouter_model] if len(exact) == 1: return exact[0] if len(exact) > 1: return None suffix = [model for model in published if openrouter_model.endswith("/" + model.model_name)] if len(suffix) == 1: return suffix[0] return None def card_match_is_ambiguous(models: dict[str, ModelConfig], openrouter_model: str) -> bool: published = [model for model in models.values() if model.provider_name != "mock"] exact = [model for model in published if model.model_name == openrouter_model] if len(exact) > 1: return True if len(exact) == 1: return False suffix = [model for model in published if openrouter_model.endswith("/" + model.model_name)] return len(suffix) > 1 def build_visitor_protocol(config: ProjectConfig, openrouter_model: str) -> VisitorProtocol: model_name = openrouter_model.strip() if not model_name: raise ValueError("OpenRouter model id is empty.") lookup_name = catalog_slug(model_name) card = match_model_card(config.models, lookup_name) ambiguous = card is None and card_match_is_ambiguous(config.models, lookup_name) caveats: list[Caveat] = [] if lookup_name != model_name: caveats.append( Caveat( level="warning", code="routing_variant", message=( f"{model_name} keeps that suffix in the model field. " f":nitro and :floor are routing variants, not rows in GET /api/v1/models. " f"The card and the catalog price are looked up on {lookup_name}. " ":nitro sorts providers by throughput and :floor sorts them by price. " ":free and :batch are real catalog rows and are not rewritten." ), ) ) if card is None: model_config = ModelConfig( model_id=safe_model_id(model_name), provider_name="openai_compatible", model_name=model_name, api_base=OPENROUTER_API_BASE, api_key_env=VISITOR_KEY_ENV, temperature=UNLISTED_TEMPERATURE, max_tokens=UNLISTED_MAX_TOKENS, timeout=UNLISTED_TIMEOUT, retry=UNLISTED_RETRY.model_copy(deep=True), rate_limits=UNLISTED_RATE_LIMITS.model_copy(deep=True), supports_system_prompt=True, supports_stream=UNLISTED_STREAM, supports_reasoning=True, provider_kwargs={}, ) if ambiguous: caveats.append( Caveat( level="warning", code="ambiguous_model_card", message=( f"{model_name} matches more than one model_name in configs/models.yaml. " "No card was copied. Generation settings are the unlisted-model modal " f"values: temperature {UNLISTED_TEMPERATURE:g}, max_tokens {UNLISTED_MAX_TOKENS}, " f"timeout {UNLISTED_TIMEOUT:g}s, stream {str(UNLISTED_STREAM).lower()}, " f"retry max_attempts {UNLISTED_RETRY.max_attempts}, empty provider_kwargs." ), ) ) else: caveats.append( Caveat( level="warning", code="no_model_card", message=( f"{model_name} has no model card in configs/models.yaml. " "Published cards do not share one max_tokens. Unlisted models use the modal " "settings of the 13 empty-provider_kwargs cards at max_tokens 8192: " f"temperature {UNLISTED_TEMPERATURE:g} (this value is on every non-mock card), " f"max_tokens {UNLISTED_MAX_TOKENS}, timeout {UNLISTED_TIMEOUT:g}s, " f"stream {str(UNLISTED_STREAM).lower()}, retry max_attempts {UNLISTED_RETRY.max_attempts}, " "empty provider_kwargs. A listed model such as gpt-5.4 uses its own card instead." ), ) ) else: model_config = card.model_copy( update={ "model_id": safe_model_id(model_name), "model_name": model_name, "api_base": OPENROUTER_API_BASE, "api_key_env": VISITOR_KEY_ENV, }, deep=True, ) caveats.append( Caveat( level="info", code="model_card_copied", message=( f"Generation settings were copied from model card {card.model_id} " f"(model_name {card.model_name}): temperature {model_config.temperature}, " f"max_tokens {model_config.max_tokens}, timeout {model_config.timeout}, " f"stream {str(bool(model_config.supports_stream)).lower()}, " f"retry max_attempts {model_config.retry.max_attempts}, " f"provider_kwargs keys {sorted(model_config.provider_kwargs) or '[]'}." ), ) ) if card.api_base and card.api_base.rstrip("/") != OPENROUTER_API_BASE: caveats.append( Caveat( level="warning", code="openrouter_host", message=( f"Card {card.model_id} calls {card.api_base}. This run calls {OPENROUTER_API_BASE}. " "The JSON body uses the same client and the copied generation settings, but the " f"model field is {model_name}, not {card.model_name}." ), ) ) if card is None: caveats.append( Caveat( level="info", code="openrouter_host", message=( f"Requests go to {OPENROUTER_API_BASE} through OpenAICompatibleClient. " "The published runs used the api_base values in configs/models.yaml." ), ) ) caveats.append( Caveat( level="warning", code="openrouter_provider_routing", message=( "OpenRouter serves this model slug from several hosts. The run limits them to the " "highest precision any host reports and tries them cheapest first (see the host note). " "allow_fallbacks defaults to true, so a host error is retried on another host at the " "same precision. require_parameters defaults to false, so a host that does not implement " "temperature or max_tokens still accepts the request and ignores those fields. " "The model field is the slug above. " "POST /api/v1/chat/completions authenticates with a Bearer token. " "Reported token counts use the model's native tokenizer." ), ) ) provider = config.providers["openai_compatible"] responses_api, _ = _preview_payload(provider, model_config, "protocol-check") card_responses: bool | None = None differences: list[str] = [] if card is not None: card_model = model_config.model_copy(update={"model_name": card.model_name}, deep=True) card_responses, card_payload = _preview_payload(provider, card_model, "protocol-check") _, visitor_payload = _preview_payload(provider, model_config, "protocol-check") differences = _payload_differences(card_payload, visitor_payload) if differences: caveats.append( Caveat( level="warning", code="request_branch_differs", message=( "OpenAICompatibleClient branches on the model string. " f"The card name {card.model_name} and the OpenRouter id {model_name} " f"do not produce the same body. Differing fields: {', '.join(differences)}. " "The run sends the OpenRouter id, because OpenRouter routes on that id. " "The client is not patched." ), ) ) return VisitorProtocol( openrouter_model=model_name, model_config=model_config, matched_card_id=None if card is None else card.model_id, matched_card_model_name=None if card is None else card.model_name, responses_api=responses_api, card_would_use_responses_api=card_responses, request_branch_differences=differences, caveats=caveats, ) def build_judge_model_config( config: ProjectConfig, openrouter_judge_model: str, *, system_one: bool = False, ) -> ModelConfig: """Build a judge ModelConfig authenticated by the single visitor OpenRouter key.""" model_name = openrouter_judge_model.strip() if not model_name: raise ValueError("Judge OpenRouter model id is empty.") if system_one: return ModelConfig( model_id=f"judge_s1_{safe_model_id(model_name)}", provider_name="openai_compatible", model_name=model_name, api_base=OPENROUTER_API_BASE, api_key_env=VISITOR_KEY_ENV, temperature=0.0, max_tokens=0, timeout=UNLISTED_TIMEOUT, retry=UNLISTED_RETRY.model_copy(deep=True), rate_limits=UNLISTED_RATE_LIMITS.model_copy(deep=True), supports_system_prompt=False, supports_stream=False, supports_reasoning=False, provider_kwargs={"system_one": True}, ) protocol = build_visitor_protocol(config, model_name) return protocol.model_config.model_copy( update={ "model_id": f"judge_{safe_model_id(model_name)}", "temperature": 0.0, # No length limit: a reasoning judge thinks before it grades, and a cap cuts off the grade. "max_tokens": None, }, deep=True, ) def build_multiturn_chat_model_config( config: ProjectConfig, openrouter_multiturn_model: str, ) -> ModelConfig: """Build a multi-turn student chat ModelConfig authenticated by the visitor OpenRouter key.""" model_name = openrouter_multiturn_model.strip() if not model_name: raise ValueError("Multi-turn OpenRouter model id is empty.") protocol = build_visitor_protocol(config, model_name) return protocol.model_config.model_copy( update={ "model_id": f"multiturn_{safe_model_id(model_name)}", "temperature": 0.0, # No length limit, for the same reason as the judge: a reasoning student needs room to think. "max_tokens": None, }, deep=True, ) def preview_standard_payload(config: ProjectConfig, model_config: ModelConfig, prompt: str) -> dict: """Build the JSON body the runner's first request would send for this prompt.""" provider = config.providers[model_config.provider_name] _, payload = _preview_payload(provider, model_config, prompt) return payload def _preview_payload( provider, model_config: ModelConfig, prompt: str, ) -> tuple[bool, dict]: client = OpenAICompatibleClient(provider, model_config) async def _build() -> tuple[bool, dict]: try: request = GenerationRequest( prompt=prompt, temperature=model_config.temperature, max_tokens=model_config.max_tokens, stream=bool(model_config.supports_stream), messages=[{"role": "user", "content": prompt}], provider_kwargs={}, ) merged = {**model_config.provider_kwargs, **request.provider_kwargs} use_responses_api = client._use_responses_api(merged) payload = client._build_payload( request, provider_kwargs=merged, use_responses_api=use_responses_api, ) return use_responses_api, payload finally: await client.aclose() return asyncio.run(_build()) def _payload_differences(card_payload: dict, visitor_payload: dict) -> list[str]: differences: list[str] = [] keys = sorted(set(card_payload) | set(visitor_payload)) for key in keys: if key == "model": continue if key in {"messages", "input"}: continue if card_payload.get(key) != visitor_payload.get(key): differences.append(key) elif key not in card_payload or key not in visitor_payload: differences.append(key) return differences