Spaces:

adelevett
/

ELBench-live

Sleeping

ELBench-live / src /elbench /dashboard /protocol.py

Redeploy ELBench-live

68670f0 verified 18 days ago

17.1 kB

"""Map an OpenRouter model id onto a source ModelConfig.
The HTTP body is still built by ``OpenAICompatibleClient``. When the id matches
one published card, generation settings are copied from that card. The ``model``
string stays the OpenRouter id, because that is the id OpenRouter routes.
"""

from __future__ import annotations

import asyncio
import re
from dataclasses import dataclass, field

from elbench.providers.openai_compatible import OpenAICompatibleClient
from elbench.schemas.config import ModelConfig, ProjectConfig, RateLimitConfig, RetryConfig
from elbench.schemas.evaluation import GenerationRequest

OPENROUTER_API_BASE = "https://openrouter.ai/api/v1"
VISITOR_KEY_ENV = "ELBENCH_VISITOR_OPENROUTER_API_KEY"
# Routing variants are not rows in GET /api/v1/models. :free and :batch are.
ROUTING_VARIANT_SUFFIXES = (":nitro", ":floor")

DEFAULT_SYSTEM_ONE_MODELS: tuple[tuple[str, str, float | None], ...] = (
    ("LiquidAI: d1-ultra", "liquid/d1-ultra", 0.042e-6),
    ("LiquidAI: d1", "liquid/d1", 0.020e-6),
    ("TypeSafe: Jev 1.14", "typesafe/jev-1.14", 0.042e-6),
    ("TypeSafe: Jev Latest", "~typesafe/jev-latest", 0.042e-6),
    ("OpenAI: GPT-6 Luna Decisions", "openai/gpt-6-luna-decisions", 0.100e-6),
    ("Perplexity: Decider V1.2", "perplexity/pplx-decider-v1.2", 0.020e-6),
    ("Perplexity: Decider Flash V1.2", "perplexity/pplx-decider-flash-v1.2", 0.010e-6),
    ("Microsoft: Microsoft-Decision-1", "microsoft/microsoft-decision-1", 0.042e-6),
    ("Cloudflare: Clef", "cloudflare/clef", 0.042e-6),
    ("Cloudflare: Clef Flash", "cloudflare/clef-flash", 0.019e-6),
    ("Cloudflare: Clef Omni", "cloudflare/clef-omni", 0.150e-6),
    ("Inception: Mercury Decide", "inception/mercury-decide", 0.020e-6),
    ("Inception: Mercury Decide (free)", "inception/mercury-decide:free", 0.0),
    ("Upstage: Solar Decide", "upstage/solar-decide", 0.050e-6),
    ("Nace.AI: Drex Max", "nace-ai/drex-max", 0.060e-6),
    ("ScaleDown: Classify-1", "scaledown/classify-1", 0.040e-6),
    ("Respan: Span-02", "respan/span-02", 0.010e-6),
    ("Baseten: Mammoth 1 (free)", "baseten/mammoth-1:free", 0.0),
)

# Modal generation settings among the 13 published openai-compatible cards that
# have empty provider_kwargs and max_tokens 8192. Temperature 0 is unanimous
# across every non-mock card. The other fields are the mode of that group, not
# a value the source defines for unlisted models.
UNLISTED_TEMPERATURE = 0.0
UNLISTED_MAX_TOKENS = 8192
UNLISTED_TIMEOUT = 240.0
UNLISTED_STREAM = False
UNLISTED_RETRY = RetryConfig(
    max_attempts=2,
    initial_delay_seconds=1.0,
    max_delay_seconds=20.0,
    backoff_multiplier=2.0,
    jitter_ratio=0.2,
    retryable_status_codes=[],
)
UNLISTED_RATE_LIMITS = RateLimitConfig(max_concurrency=100)


@dataclass(slots=True)
class Caveat:
    level: str
    code: str
    message: str


@dataclass(slots=True)
class VisitorProtocol:
    openrouter_model: str
    model_config: ModelConfig
    matched_card_id: str | None
    matched_card_model_name: str | None
    responses_api: bool
    card_would_use_responses_api: bool | None
    request_branch_differences: list[str] = field(default_factory=list)
    caveats: list[Caveat] = field(default_factory=list)


def catalog_slug(openrouter_model: str) -> str:
    """Slug whose card and catalog price apply.
    ``:nitro`` and ``:floor`` only change provider sort. ``:free`` and ``:batch``
    are separate catalog rows and stay intact.
    """
    model = openrouter_model.strip()
    for suffix in ROUTING_VARIANT_SUFFIXES:
        if model.endswith(suffix):
            return model[: -len(suffix)]
    return model


def safe_model_id(openrouter_model: str) -> str:
    """Filesystem-safe id. The basic-education runtime uses model_id as a path segment."""
    cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", openrouter_model).strip("._-")
    if not cleaned:
        cleaned = "model"
    return f"openrouter_{cleaned[:80]}"


def match_model_card(models: dict[str, ModelConfig], openrouter_model: str) -> ModelConfig | None:
    published = [model for model in models.values() if model.provider_name != "mock"]
    exact = [model for model in published if model.model_name == openrouter_model]
    if len(exact) == 1:
        return exact[0]
    if len(exact) > 1:
        return None
    suffix = [model for model in published if openrouter_model.endswith("/" + model.model_name)]
    if len(suffix) == 1:
        return suffix[0]
    return None


def card_match_is_ambiguous(models: dict[str, ModelConfig], openrouter_model: str) -> bool:
    published = [model for model in models.values() if model.provider_name != "mock"]
    exact = [model for model in published if model.model_name == openrouter_model]
    if len(exact) > 1:
        return True
    if len(exact) == 1:
        return False
    suffix = [model for model in published if openrouter_model.endswith("/" + model.model_name)]
    return len(suffix) > 1


def build_visitor_protocol(config: ProjectConfig, openrouter_model: str) -> VisitorProtocol:
    model_name = openrouter_model.strip()
    if not model_name:
        raise ValueError("OpenRouter model id is empty.")

    lookup_name = catalog_slug(model_name)
    card = match_model_card(config.models, lookup_name)
    ambiguous = card is None and card_match_is_ambiguous(config.models, lookup_name)
    caveats: list[Caveat] = []
    if lookup_name != model_name:
        caveats.append(
            Caveat(
                level="warning",
                code="routing_variant",
                message=(
                    f"{model_name} keeps that suffix in the model field. "
                    f":nitro and :floor are routing variants, not rows in GET /api/v1/models. "
                    f"The card and the catalog price are looked up on {lookup_name}. "
                    ":nitro sorts providers by throughput and :floor sorts them by price. "
                    ":free and :batch are real catalog rows and are not rewritten."
                ),
            )
        )

    if card is None:
        model_config = ModelConfig(
            model_id=safe_model_id(model_name),
            provider_name="openai_compatible",
            model_name=model_name,
            api_base=OPENROUTER_API_BASE,
            api_key_env=VISITOR_KEY_ENV,
            temperature=UNLISTED_TEMPERATURE,
            max_tokens=UNLISTED_MAX_TOKENS,
            timeout=UNLISTED_TIMEOUT,
            retry=UNLISTED_RETRY.model_copy(deep=True),
            rate_limits=UNLISTED_RATE_LIMITS.model_copy(deep=True),
            supports_system_prompt=True,
            supports_stream=UNLISTED_STREAM,
            supports_reasoning=True,
            provider_kwargs={},
        )
        if ambiguous:
            caveats.append(
                Caveat(
                    level="warning",
                    code="ambiguous_model_card",
                    message=(
                        f"{model_name} matches more than one model_name in configs/models.yaml. "
                        "No card was copied. Generation settings are the unlisted-model modal "
                        f"values: temperature {UNLISTED_TEMPERATURE:g}, max_tokens {UNLISTED_MAX_TOKENS}, "
                        f"timeout {UNLISTED_TIMEOUT:g}s, stream {str(UNLISTED_STREAM).lower()}, "
                        f"retry max_attempts {UNLISTED_RETRY.max_attempts}, empty provider_kwargs."
                    ),
                )
            )
        else:
            caveats.append(
                Caveat(
                    level="warning",
                    code="no_model_card",
                    message=(
                        f"{model_name} has no model card in configs/models.yaml. "
                        "Published cards do not share one max_tokens. Unlisted models use the modal "
                        "settings of the 13 empty-provider_kwargs cards at max_tokens 8192: "
                        f"temperature {UNLISTED_TEMPERATURE:g} (this value is on every non-mock card), "
                        f"max_tokens {UNLISTED_MAX_TOKENS}, timeout {UNLISTED_TIMEOUT:g}s, "
                        f"stream {str(UNLISTED_STREAM).lower()}, retry max_attempts {UNLISTED_RETRY.max_attempts}, "
                        "empty provider_kwargs. A listed model such as gpt-5.4 uses its own card instead."
                    ),
                )
            )
    else:
        model_config = card.model_copy(
            update={
                "model_id": safe_model_id(model_name),
                "model_name": model_name,
                "api_base": OPENROUTER_API_BASE,
                "api_key_env": VISITOR_KEY_ENV,
            },
            deep=True,
        )
        caveats.append(
            Caveat(
                level="info",
                code="model_card_copied",
                message=(
                    f"Generation settings were copied from model card {card.model_id} "
                    f"(model_name {card.model_name}): temperature {model_config.temperature}, "
                    f"max_tokens {model_config.max_tokens}, timeout {model_config.timeout}, "
                    f"stream {str(bool(model_config.supports_stream)).lower()}, "
                    f"retry max_attempts {model_config.retry.max_attempts}, "
                    f"provider_kwargs keys {sorted(model_config.provider_kwargs) or '[]'}."
                ),
            )
        )
        if card.api_base and card.api_base.rstrip("/") != OPENROUTER_API_BASE:
            caveats.append(
                Caveat(
                    level="warning",
                    code="openrouter_host",
                    message=(
                        f"Card {card.model_id} calls {card.api_base}. This run calls {OPENROUTER_API_BASE}. "
                        "The JSON body uses the same client and the copied generation settings, but the "
                        f"model field is {model_name}, not {card.model_name}."
                    ),
                )
            )

    if card is None:
        caveats.append(
            Caveat(
                level="info",
                code="openrouter_host",
                message=(
                    f"Requests go to {OPENROUTER_API_BASE} through OpenAICompatibleClient. "
                    "The published runs used the api_base values in configs/models.yaml."
                ),
            )
        )

    caveats.append(
        Caveat(
            level="warning",
            code="openrouter_provider_routing",
            message=(
                "OpenRouter serves this model slug from several hosts. The run limits them to the "
                "highest precision any host reports and tries them cheapest first (see the host note). "
                "allow_fallbacks defaults to true, so a host error is retried on another host at the "
                "same precision. require_parameters defaults to false, so a host that does not implement "
                "temperature or max_tokens still accepts the request and ignores those fields. "
                "The model field is the slug above. "
                "POST /api/v1/chat/completions authenticates with a Bearer token. "
                "Reported token counts use the model's native tokenizer."
            ),
        )
    )

    provider = config.providers["openai_compatible"]
    responses_api, _ = _preview_payload(provider, model_config, "protocol-check")
    card_responses: bool | None = None
    differences: list[str] = []
    if card is not None:
        card_model = model_config.model_copy(update={"model_name": card.model_name}, deep=True)
        card_responses, card_payload = _preview_payload(provider, card_model, "protocol-check")
        _, visitor_payload = _preview_payload(provider, model_config, "protocol-check")
        differences = _payload_differences(card_payload, visitor_payload)
        if differences:
            caveats.append(
                Caveat(
                    level="warning",
                    code="request_branch_differs",
                    message=(
                        "OpenAICompatibleClient branches on the model string. "
                        f"The card name {card.model_name} and the OpenRouter id {model_name} "
                        f"do not produce the same body. Differing fields: {', '.join(differences)}. "
                        "The run sends the OpenRouter id, because OpenRouter routes on that id. "
                        "The client is not patched."
                    ),
                )
            )

    return VisitorProtocol(
        openrouter_model=model_name,
        model_config=model_config,
        matched_card_id=None if card is None else card.model_id,
        matched_card_model_name=None if card is None else card.model_name,
        responses_api=responses_api,
        card_would_use_responses_api=card_responses,
        request_branch_differences=differences,
        caveats=caveats,
    )


def build_judge_model_config(
    config: ProjectConfig,
    openrouter_judge_model: str,
    *,
    system_one: bool = False,
) -> ModelConfig:
    """Build a judge ModelConfig authenticated by the single visitor OpenRouter key."""
    model_name = openrouter_judge_model.strip()
    if not model_name:
        raise ValueError("Judge OpenRouter model id is empty.")
    if system_one:
        return ModelConfig(
            model_id=f"judge_s1_{safe_model_id(model_name)}",
            provider_name="openai_compatible",
            model_name=model_name,
            api_base=OPENROUTER_API_BASE,
            api_key_env=VISITOR_KEY_ENV,
            temperature=0.0,
            max_tokens=0,
            timeout=UNLISTED_TIMEOUT,
            retry=UNLISTED_RETRY.model_copy(deep=True),
            rate_limits=UNLISTED_RATE_LIMITS.model_copy(deep=True),
            supports_system_prompt=False,
            supports_stream=False,
            supports_reasoning=False,
            provider_kwargs={"system_one": True},
        )
    protocol = build_visitor_protocol(config, model_name)
    return protocol.model_config.model_copy(
        update={
            "model_id": f"judge_{safe_model_id(model_name)}",
            "temperature": 0.0,
            # No length limit: a reasoning judge thinks before it grades, and a cap cuts off the grade.
            "max_tokens": None,
        },
        deep=True,
    )


def build_multiturn_chat_model_config(
    config: ProjectConfig,
    openrouter_multiturn_model: str,
) -> ModelConfig:
    """Build a multi-turn student chat ModelConfig authenticated by the visitor OpenRouter key."""
    model_name = openrouter_multiturn_model.strip()
    if not model_name:
        raise ValueError("Multi-turn OpenRouter model id is empty.")
    protocol = build_visitor_protocol(config, model_name)
    return protocol.model_config.model_copy(
        update={
            "model_id": f"multiturn_{safe_model_id(model_name)}",
            "temperature": 0.0,
            # No length limit, for the same reason as the judge: a reasoning student needs room to think.
            "max_tokens": None,
        },
        deep=True,
    )


def preview_standard_payload(config: ProjectConfig, model_config: ModelConfig, prompt: str) -> dict:
    """Build the JSON body the runner's first request would send for this prompt."""
    provider = config.providers[model_config.provider_name]
    _, payload = _preview_payload(provider, model_config, prompt)
    return payload


def _preview_payload(
    provider,
    model_config: ModelConfig,
    prompt: str,
) -> tuple[bool, dict]:
    client = OpenAICompatibleClient(provider, model_config)

    async def _build() -> tuple[bool, dict]:
        try:
            request = GenerationRequest(
                prompt=prompt,
                temperature=model_config.temperature,
                max_tokens=model_config.max_tokens,
                stream=bool(model_config.supports_stream),
                messages=[{"role": "user", "content": prompt}],
                provider_kwargs={},
            )
            merged = {**model_config.provider_kwargs, **request.provider_kwargs}
            use_responses_api = client._use_responses_api(merged)
            payload = client._build_payload(
                request,
                provider_kwargs=merged,
                use_responses_api=use_responses_api,
            )
            return use_responses_api, payload
        finally:
            await client.aclose()

    return asyncio.run(_build())


def _payload_differences(card_payload: dict, visitor_payload: dict) -> list[str]:
    differences: list[str] = []
    keys = sorted(set(card_payload) | set(visitor_payload))
    for key in keys:
        if key == "model":
            continue
        if key in {"messages", "input"}:
            continue
        if card_payload.get(key) != visitor_payload.get(key):
            differences.append(key)
        elif key not in card_payload or key not in visitor_payload:
            differences.append(key)
    return differences

	"""Map an OpenRouter model id onto a source ModelConfig.

	The HTTP body is still built by ``OpenAICompatibleClient``. When the id matches
	one published card, generation settings are copied from that card. The ``model``
	string stays the OpenRouter id, because that is the id OpenRouter routes.
	"""

	from __future__ import annotations

	import asyncio
	import re
	from dataclasses import dataclass, field

	from elbench.providers.openai_compatible import OpenAICompatibleClient
	from elbench.schemas.config import ModelConfig, ProjectConfig, RateLimitConfig, RetryConfig
	from elbench.schemas.evaluation import GenerationRequest

	OPENROUTER_API_BASE = "https://openrouter.ai/api/v1"
	VISITOR_KEY_ENV = "ELBENCH_VISITOR_OPENROUTER_API_KEY"
	# Routing variants are not rows in GET /api/v1/models. :free and :batch are.
	ROUTING_VARIANT_SUFFIXES = (":nitro", ":floor")

	DEFAULT_SYSTEM_ONE_MODELS: tuple[tuple[str, str, float \| None], ...] = (
	("LiquidAI: d1-ultra", "liquid/d1-ultra", 0.042e-6),
	("LiquidAI: d1", "liquid/d1", 0.020e-6),
	("TypeSafe: Jev 1.14", "typesafe/jev-1.14", 0.042e-6),
	("TypeSafe: Jev Latest", "~typesafe/jev-latest", 0.042e-6),
	("OpenAI: GPT-6 Luna Decisions", "openai/gpt-6-luna-decisions", 0.100e-6),
	("Perplexity: Decider V1.2", "perplexity/pplx-decider-v1.2", 0.020e-6),
	("Perplexity: Decider Flash V1.2", "perplexity/pplx-decider-flash-v1.2", 0.010e-6),
	("Microsoft: Microsoft-Decision-1", "microsoft/microsoft-decision-1", 0.042e-6),
	("Cloudflare: Clef", "cloudflare/clef", 0.042e-6),
	("Cloudflare: Clef Flash", "cloudflare/clef-flash", 0.019e-6),
	("Cloudflare: Clef Omni", "cloudflare/clef-omni", 0.150e-6),
	("Inception: Mercury Decide", "inception/mercury-decide", 0.020e-6),
	("Inception: Mercury Decide (free)", "inception/mercury-decide:free", 0.0),
	("Upstage: Solar Decide", "upstage/solar-decide", 0.050e-6),
	("Nace.AI: Drex Max", "nace-ai/drex-max", 0.060e-6),
	("ScaleDown: Classify-1", "scaledown/classify-1", 0.040e-6),
	("Respan: Span-02", "respan/span-02", 0.010e-6),
	("Baseten: Mammoth 1 (free)", "baseten/mammoth-1:free", 0.0),
	)

	# Modal generation settings among the 13 published openai-compatible cards that
	# have empty provider_kwargs and max_tokens 8192. Temperature 0 is unanimous
	# across every non-mock card. The other fields are the mode of that group, not
	# a value the source defines for unlisted models.
	UNLISTED_TEMPERATURE = 0.0
	UNLISTED_MAX_TOKENS = 8192
	UNLISTED_TIMEOUT = 240.0
	UNLISTED_STREAM = False
	UNLISTED_RETRY = RetryConfig(
	max_attempts=2,
	initial_delay_seconds=1.0,
	max_delay_seconds=20.0,
	backoff_multiplier=2.0,
	jitter_ratio=0.2,
	retryable_status_codes=[],
	)
	UNLISTED_RATE_LIMITS = RateLimitConfig(max_concurrency=100)


	@dataclass(slots=True)
	class Caveat:
	level: str
	code: str
	message: str


	@dataclass(slots=True)
	class VisitorProtocol:
	openrouter_model: str
	model_config: ModelConfig
	matched_card_id: str \| None
	matched_card_model_name: str \| None
	responses_api: bool
	card_would_use_responses_api: bool \| None
	request_branch_differences: list[str] = field(default_factory=list)
	caveats: list[Caveat] = field(default_factory=list)


	def catalog_slug(openrouter_model: str) -> str:
	"""Slug whose card and catalog price apply.

	``:nitro`` and ``:floor`` only change provider sort. ``:free`` and ``:batch``
	are separate catalog rows and stay intact.
	"""
	model = openrouter_model.strip()
	for suffix in ROUTING_VARIANT_SUFFIXES:
	if model.endswith(suffix):
	return model[: -len(suffix)]
	return model


	def safe_model_id(openrouter_model: str) -> str:
	"""Filesystem-safe id. The basic-education runtime uses model_id as a path segment."""
	cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", openrouter_model).strip("._-")
	if not cleaned:
	cleaned = "model"
	return f"openrouter_{cleaned[:80]}"


	def match_model_card(models: dict[str, ModelConfig], openrouter_model: str) -> ModelConfig \| None:
	published = [model for model in models.values() if model.provider_name != "mock"]
	exact = [model for model in published if model.model_name == openrouter_model]
	if len(exact) == 1:
	return exact[0]
	if len(exact) > 1:
	return None
	suffix = [model for model in published if openrouter_model.endswith("/" + model.model_name)]
	if len(suffix) == 1:
	return suffix[0]
	return None


	def card_match_is_ambiguous(models: dict[str, ModelConfig], openrouter_model: str) -> bool:
	published = [model for model in models.values() if model.provider_name != "mock"]
	exact = [model for model in published if model.model_name == openrouter_model]
	if len(exact) > 1:
	return True
	if len(exact) == 1:
	return False
	suffix = [model for model in published if openrouter_model.endswith("/" + model.model_name)]
	return len(suffix) > 1


	def build_visitor_protocol(config: ProjectConfig, openrouter_model: str) -> VisitorProtocol:
	model_name = openrouter_model.strip()
	if not model_name:
	raise ValueError("OpenRouter model id is empty.")

	lookup_name = catalog_slug(model_name)
	card = match_model_card(config.models, lookup_name)
	ambiguous = card is None and card_match_is_ambiguous(config.models, lookup_name)
	caveats: list[Caveat] = []
	if lookup_name != model_name:
	caveats.append(
	Caveat(
	level="warning",
	code="routing_variant",
	message=(
	f"{model_name} keeps that suffix in the model field. "
	f":nitro and :floor are routing variants, not rows in GET /api/v1/models. "
	f"The card and the catalog price are looked up on {lookup_name}. "
	":nitro sorts providers by throughput and :floor sorts them by price. "
	":free and :batch are real catalog rows and are not rewritten."
	),
	)
	)

	if card is None:
	model_config = ModelConfig(
	model_id=safe_model_id(model_name),
	provider_name="openai_compatible",
	model_name=model_name,
	api_base=OPENROUTER_API_BASE,
	api_key_env=VISITOR_KEY_ENV,
	temperature=UNLISTED_TEMPERATURE,
	max_tokens=UNLISTED_MAX_TOKENS,
	timeout=UNLISTED_TIMEOUT,
	retry=UNLISTED_RETRY.model_copy(deep=True),
	rate_limits=UNLISTED_RATE_LIMITS.model_copy(deep=True),
	supports_system_prompt=True,
	supports_stream=UNLISTED_STREAM,
	supports_reasoning=True,
	provider_kwargs={},
	)
	if ambiguous:
	caveats.append(
	Caveat(
	level="warning",
	code="ambiguous_model_card",
	message=(
	f"{model_name} matches more than one model_name in configs/models.yaml. "
	"No card was copied. Generation settings are the unlisted-model modal "
	f"values: temperature {UNLISTED_TEMPERATURE:g}, max_tokens {UNLISTED_MAX_TOKENS}, "
	f"timeout {UNLISTED_TIMEOUT:g}s, stream {str(UNLISTED_STREAM).lower()}, "
	f"retry max_attempts {UNLISTED_RETRY.max_attempts}, empty provider_kwargs."
	),
	)
	)
	else:
	caveats.append(
	Caveat(
	level="warning",
	code="no_model_card",
	message=(
	f"{model_name} has no model card in configs/models.yaml. "
	"Published cards do not share one max_tokens. Unlisted models use the modal "
	"settings of the 13 empty-provider_kwargs cards at max_tokens 8192: "
	f"temperature {UNLISTED_TEMPERATURE:g} (this value is on every non-mock card), "
	f"max_tokens {UNLISTED_MAX_TOKENS}, timeout {UNLISTED_TIMEOUT:g}s, "
	f"stream {str(UNLISTED_STREAM).lower()}, retry max_attempts {UNLISTED_RETRY.max_attempts}, "
	"empty provider_kwargs. A listed model such as gpt-5.4 uses its own card instead."
	),
	)
	)
	else:
	model_config = card.model_copy(
	update={
	"model_id": safe_model_id(model_name),
	"model_name": model_name,
	"api_base": OPENROUTER_API_BASE,
	"api_key_env": VISITOR_KEY_ENV,
	},
	deep=True,
	)
	caveats.append(
	Caveat(
	level="info",
	code="model_card_copied",
	message=(
	f"Generation settings were copied from model card {card.model_id} "
	f"(model_name {card.model_name}): temperature {model_config.temperature}, "
	f"max_tokens {model_config.max_tokens}, timeout {model_config.timeout}, "
	f"stream {str(bool(model_config.supports_stream)).lower()}, "
	f"retry max_attempts {model_config.retry.max_attempts}, "
	f"provider_kwargs keys {sorted(model_config.provider_kwargs) or '[]'}."
	),
	)
	)
	if card.api_base and card.api_base.rstrip("/") != OPENROUTER_API_BASE:
	caveats.append(
	Caveat(
	level="warning",
	code="openrouter_host",
	message=(
	f"Card {card.model_id} calls {card.api_base}. This run calls {OPENROUTER_API_BASE}. "
	"The JSON body uses the same client and the copied generation settings, but the "
	f"model field is {model_name}, not {card.model_name}."
	),
	)
	)

	if card is None:
	caveats.append(
	Caveat(
	level="info",
	code="openrouter_host",
	message=(
	f"Requests go to {OPENROUTER_API_BASE} through OpenAICompatibleClient. "
	"The published runs used the api_base values in configs/models.yaml."
	),
	)
	)

	caveats.append(
	Caveat(
	level="warning",
	code="openrouter_provider_routing",
	message=(
	"OpenRouter serves this model slug from several hosts. The run limits them to the "
	"highest precision any host reports and tries them cheapest first (see the host note). "
	"allow_fallbacks defaults to true, so a host error is retried on another host at the "
	"same precision. require_parameters defaults to false, so a host that does not implement "
	"temperature or max_tokens still accepts the request and ignores those fields. "
	"The model field is the slug above. "
	"POST /api/v1/chat/completions authenticates with a Bearer token. "
	"Reported token counts use the model's native tokenizer."
	),
	)
	)

	provider = config.providers["openai_compatible"]
	responses_api, _ = _preview_payload(provider, model_config, "protocol-check")
	card_responses: bool \| None = None
	differences: list[str] = []
	if card is not None:
	card_model = model_config.model_copy(update={"model_name": card.model_name}, deep=True)
	card_responses, card_payload = _preview_payload(provider, card_model, "protocol-check")
	_, visitor_payload = _preview_payload(provider, model_config, "protocol-check")
	differences = _payload_differences(card_payload, visitor_payload)
	if differences:
	caveats.append(
	Caveat(
	level="warning",
	code="request_branch_differs",
	message=(
	"OpenAICompatibleClient branches on the model string. "
	f"The card name {card.model_name} and the OpenRouter id {model_name} "
	f"do not produce the same body. Differing fields: {', '.join(differences)}. "
	"The run sends the OpenRouter id, because OpenRouter routes on that id. "
	"The client is not patched."
	),
	)
	)

	return VisitorProtocol(
	openrouter_model=model_name,
	model_config=model_config,
	matched_card_id=None if card is None else card.model_id,
	matched_card_model_name=None if card is None else card.model_name,
	responses_api=responses_api,
	card_would_use_responses_api=card_responses,
	request_branch_differences=differences,
	caveats=caveats,
	)


	def build_judge_model_config(
	config: ProjectConfig,
	openrouter_judge_model: str,
	*,
	system_one: bool = False,
	) -> ModelConfig:
	"""Build a judge ModelConfig authenticated by the single visitor OpenRouter key."""
	model_name = openrouter_judge_model.strip()
	if not model_name:
	raise ValueError("Judge OpenRouter model id is empty.")
	if system_one:
	return ModelConfig(
	model_id=f"judge_s1_{safe_model_id(model_name)}",
	provider_name="openai_compatible",
	model_name=model_name,
	api_base=OPENROUTER_API_BASE,
	api_key_env=VISITOR_KEY_ENV,
	temperature=0.0,
	max_tokens=0,
	timeout=UNLISTED_TIMEOUT,
	retry=UNLISTED_RETRY.model_copy(deep=True),
	rate_limits=UNLISTED_RATE_LIMITS.model_copy(deep=True),
	supports_system_prompt=False,
	supports_stream=False,
	supports_reasoning=False,
	provider_kwargs={"system_one": True},
	)
	protocol = build_visitor_protocol(config, model_name)
	return protocol.model_config.model_copy(
	update={
	"model_id": f"judge_{safe_model_id(model_name)}",
	"temperature": 0.0,
	# No length limit: a reasoning judge thinks before it grades, and a cap cuts off the grade.
	"max_tokens": None,
	},
	deep=True,
	)


	def build_multiturn_chat_model_config(
	config: ProjectConfig,
	openrouter_multiturn_model: str,
	) -> ModelConfig:
	"""Build a multi-turn student chat ModelConfig authenticated by the visitor OpenRouter key."""
	model_name = openrouter_multiturn_model.strip()
	if not model_name:
	raise ValueError("Multi-turn OpenRouter model id is empty.")
	protocol = build_visitor_protocol(config, model_name)
	return protocol.model_config.model_copy(
	update={
	"model_id": f"multiturn_{safe_model_id(model_name)}",
	"temperature": 0.0,
	# No length limit, for the same reason as the judge: a reasoning student needs room to think.
	"max_tokens": None,
	},
	deep=True,
	)


	def preview_standard_payload(config: ProjectConfig, model_config: ModelConfig, prompt: str) -> dict:
	"""Build the JSON body the runner's first request would send for this prompt."""
	provider = config.providers[model_config.provider_name]
	_, payload = _preview_payload(provider, model_config, prompt)
	return payload


	def _preview_payload(
	provider,
	model_config: ModelConfig,
	prompt: str,
	) -> tuple[bool, dict]:
	client = OpenAICompatibleClient(provider, model_config)

	async def _build() -> tuple[bool, dict]:
	try:
	request = GenerationRequest(
	prompt=prompt,
	temperature=model_config.temperature,
	max_tokens=model_config.max_tokens,
	stream=bool(model_config.supports_stream),
	messages=[{"role": "user", "content": prompt}],
	provider_kwargs={},
	)
	merged = {model_config.provider_kwargs, request.provider_kwargs}
	use_responses_api = client._use_responses_api(merged)
	payload = client._build_payload(
	request,
	provider_kwargs=merged,
	use_responses_api=use_responses_api,
	)
	return use_responses_api, payload
	finally:
	await client.aclose()

	return asyncio.run(_build())


	def _payload_differences(card_payload: dict, visitor_payload: dict) -> list[str]:
	differences: list[str] = []
	keys = sorted(set(card_payload) \| set(visitor_payload))
	for key in keys:
	if key == "model":
	continue
	if key in {"messages", "input"}:
	continue
	if card_payload.get(key) != visitor_payload.get(key):
	differences.append(key)
	elif key not in card_payload or key not in visitor_payload:
	differences.append(key)
	return differences