Spaces:
Sleeping
Sleeping
File size: 35,748 Bytes
5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac b312acd 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 b312acd 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac 68670f0 5db30ac | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 | """Selection caveats and a cost range for one visitor run.
Sample order, prompt text, and the token floor follow the runner:
``FileRegistry.resolve``, ``LoaderFactory``, ``augment_prompt_for_format``,
and ``len(prompt) // 4``.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from elbench.execution.basic_education import (
DEFAULT_BASIC_MODULE_NAME,
DEFAULT_MULTI_TURN_RECURSION_LIMIT,
BasicEducationExecutor,
count_runtime_tasks,
)
from elbench.execution.preflight import PreflightRunner
from elbench.execution.response_format import augment_prompt_for_format
from elbench.execution.runner import BenchmarkRunner
from elbench.execution.selection import resolve_module_selection
from elbench.loaders import LoaderFactory
from elbench.registry import FileRegistry
from elbench.schemas.config import ProjectConfig
from elbench.schemas.evaluation import Sample
from elbench.dashboard.catalog import Catalog, CatalogFile
from elbench.dashboard.english import english_file, english_module, english_preflight_message
from elbench.dashboard.leaderboard import CHINA_SPECIFIC_FILES, compute_plan_fingerprint
from elbench.dashboard.protocol import (
VISITOR_KEY_ENV,
Caveat,
VisitorProtocol,
build_judge_model_config,
build_multiturn_chat_model_config,
build_visitor_protocol,
)
from elbench.dashboard.routing import Route
from elbench.schemas.config import ModelConfig
from elbench.utils.secrets import get_api_key
# High-end assumption for one judge grade, thinking included. Not sent as a limit.
JUDGE_HIGH_OUTPUT_TOKENS = 8192
@dataclass(frozen=True, slots=True)
class RunSelection:
modules: frozenset[str]
source_files: frozenset[str]
dimensions: frozenset[str]
max_samples: int | None
max_samples_per_file: int | None = None
@dataclass(slots=True)
class Plan:
protocol: VisitorProtocol
selection: RunSelection
caveats: list[Caveat]
standard_requests: int
standard_low_input_tokens: int
standard_high_input_tokens: int
standard_output_cap_tokens: int
judge_calls: int
judge_model_id: str | None
judge_api_base: str | None
basic_tasks: int
basic_target_calls_low: int
basic_target_calls_high: int
max_tokens: int
prompt_usd_per_token: float | None
completion_usd_per_token: float | None
low_usd: float | None
high_usd: float | None
preflight_ok: bool
blocked: bool
needs_price_ack: bool
file_counts: list[tuple[str, int]] = field(default_factory=list)
judge_model_config: ModelConfig | None = None
use_system_one_judge: bool = False
multiturn_model_config: ModelConfig | None = None
use_multiturn_chat_model: bool = False
config_hash: str = ""
is_full_benchmark: bool = False
# Every question outside the region-specific sets is selected (the default "core" scope).
covers_core: bool = False
judge_prices: tuple[float, float] | None = None
student_prices: tuple[float, float] | None = None
# (low, high) USD per component: "target", "judge". Missing means no estimate.
component_usd: dict[str, tuple[float, float]] = field(default_factory=dict)
# Host choice per role ("target", "judge", "student") that the request bodies carry.
routes: dict[str, Route] = field(default_factory=dict)
def error_messages(self) -> list[str]:
return [caveat.message for caveat in self.caveats if caveat.level == "error"]
def build_plan(
catalog: Catalog,
*,
openrouter_model: str,
modules: set[str],
source_files: set[str],
dimensions: set[str] | None,
max_samples: int | None,
prompt_usd_per_token: float | None,
completion_usd_per_token: float | None,
max_concurrency: int,
max_samples_per_file: int | None = None,
judge_openrouter_model: str | None = None,
use_system_one_judge: bool = False,
judge_prompt_usd_per_token: float | None = None,
judge_completion_usd_per_token: float | None = None,
use_multiturn_chat_model: bool = False,
multiturn_openrouter_model: str | None = None,
multiturn_prompt_usd_per_token: float | None = None,
multiturn_completion_usd_per_token: float | None = None,
routes: dict[str, Route] | None = None,
) -> Plan:
protocol = build_visitor_protocol(catalog.config, openrouter_model)
config = catalog.config.model_copy(deep=True)
config.models[protocol.model_config.model_id] = protocol.model_config
selection = RunSelection(
modules=frozenset(modules),
source_files=frozenset(source_files),
dimensions=frozenset(dimensions or ()),
max_samples=max_samples,
max_samples_per_file=max_samples_per_file,
)
caveats = list(protocol.caveats)
judge_model_config: ModelConfig | None = None
cleaned_judge = (judge_openrouter_model or "").strip()
if cleaned_judge:
judge_model_config = build_judge_model_config(
config,
cleaned_judge,
system_one=bool(use_system_one_judge),
)
config.models[judge_model_config.model_id] = judge_model_config
config.judges.default_judge_model_id = judge_model_config.model_id
config.judges.basic_education_evaluator_model_id = judge_model_config.model_id
config.judges.basic_education_test_endpoint_model_id = judge_model_config.model_id
multiturn_model_config: ModelConfig | None = None
cleaned_multiturn = (multiturn_openrouter_model or "").strip()
if use_multiturn_chat_model:
if not cleaned_multiturn:
caveats.append(
Caveat(
level="error",
code="multiturn_model_missing",
message=(
"Multi-turn chat model endpoint is enabled, but no OpenRouter model id "
"was selected for the multi-turn endpoint."
),
)
)
else:
multiturn_model_config = build_multiturn_chat_model_config(config, cleaned_multiturn)
config.models[multiturn_model_config.model_id] = multiturn_model_config
config.judges.basic_education_test_endpoint_model_id = multiturn_model_config.model_id
caveats.append(
Caveat(
level="info",
code="multiturn_chat_on_visitor_key",
message=(
f"Multi-turn student dialogue (`stu`) uses OpenRouter chat model "
f"`{multiturn_model_config.model_name}` via the visitor's single OpenRouter key."
),
)
)
elif (
use_system_one_judge
and DEFAULT_BASIC_MODULE_NAME in selection.modules
and "config_guided_task.yaml" in selection.source_files
):
caveats.append(
Caveat(
level="error",
code="multiturn_endpoint_required_for_system_one",
message=(
"Scenario `config_guided_task.yaml` is multi-turn and requires a conversational "
"student model (`stu`). System 1 decision models have 0 output tokens and cannot "
"generate student dialogue turns. Check 'Opt into multi-turn chat model endpoint' "
"and select an OpenRouter chat model."
),
)
)
if protocol.openrouter_model.endswith(":free"):
caveats.append(
Caveat(
level="warning",
code="free_model_rate_limit",
message=(
"This id ends in :free. OpenRouter caps those variants at 20 requests per minute, "
"and at 50 requests per UTC day until the account has purchased at least 10 credits, "
"then 1,000 per day. Paid slugs are not under that cap. They are limited by credit "
"balance, the in-flight spending budget, the upstream provider, and DDoS protection. "
"GET /api/v1/key reports limit_remaining and free_model_daily_requests."
),
)
)
applied_routes = _apply_routes(
routes or {},
target=protocol.model_config,
judge=None if use_system_one_judge else judge_model_config,
student=multiturn_model_config,
)
if applied_routes:
caveats.append(
Caveat(
level="info",
code="openrouter_host_choice",
message=" ".join(route.describe() for route in applied_routes.values()),
)
)
caveats.extend(_selection_caveats(catalog, selection, max_concurrency))
samples = iter_standard_samples(config, selection)
file_counts: dict[str, int] = {}
low_input = 0
high_input = 0
judge_calls = 0
for sample in samples:
file_counts[sample.source_file] = file_counts.get(sample.source_file, 0) + 1
prompt = augment_prompt_for_format(sample, sample.prompt)
chars = len(prompt)
low_input += max(1, chars // 4)
high_input += max(1, chars)
if _task_mode(config, sample.task) == "llm":
judge_calls += 1
basic_tasks, basic_low, basic_high, basic_caveats = _basic_education_bounds(
config,
protocol.model_config,
selection,
standard_loaded=len(samples),
)
caveats.extend(basic_caveats)
max_tokens = int(protocol.model_config.max_tokens or 0)
output_cap = len(samples) * max_tokens
judge_model_id, judge_api_base = _judge_endpoint(config)
price_ok = (
prompt_usd_per_token is not None
and completion_usd_per_token is not None
and prompt_usd_per_token >= 0
and completion_usd_per_token >= 0
)
low_usd = None
high_usd = None
component_usd: dict[str, tuple[float, float]] = {}
if price_ok:
low_usd = (low_input * prompt_usd_per_token) + (basic_low * max_tokens * completion_usd_per_token)
high_usd = (high_input * prompt_usd_per_token) + (
(output_cap + (basic_high * max_tokens)) * completion_usd_per_token
)
component_usd["target"] = (low_usd, high_usd)
caveats.append(
Caveat(
level="info",
code="cost_range",
message=(
"The low dollar figure prices input tokens with the runner's estimator, "
"len(prompt) // 4, and prices no standard-module output. "
"The high figure prices one token per prompt character and output at the "
"request max_tokens cap. OpenRouter's usage object counts the model's native tokenizer, "
"not this estimator. Paid requests in flight are also held against an in-flight spending "
"budget estimated from max_tokens. "
"Format retries, answer-length retries, and HTTP retries are extra and are not included. "
"Basic-education input tokens are not included, because that runtime assembles its own prompts."
),
)
)
else:
caveats.append(
Caveat(
level="warning",
code="price_unknown",
message=(
f"OpenRouter did not return a token price for {protocol.openrouter_model}. "
"There is no dollar estimate. Running requires an explicit confirmation."
),
)
)
if judge_calls:
if judge_model_config is not None:
endpoint_suffix = "/systemone" if use_system_one_judge else "/chat/completions"
judge_kind = "System 1 decision-model" if use_system_one_judge else "OpenRouter chat"
caveats.append(
Caveat(
level="info",
code="judge_on_visitor_key",
message=(
f"This selection makes {judge_calls} {judge_kind} judge calls through JudgeRouter "
f"to `{judge_model_config.model_name}` at `{judge_model_config.api_base}{endpoint_suffix}` "
"using the visitor's single OpenRouter API key."
),
)
)
if (
low_usd is not None
and high_usd is not None
and judge_prompt_usd_per_token is not None
and judge_prompt_usd_per_token >= 0
):
# Each judge call sends state + rubric (~300 tokens low, ~1024 tokens high). The judge has no
# length limit, so the high end assumes JUDGE_HIGH_OUTPUT_TOKENS of output, thinking included.
judge_low = judge_calls * 300 * judge_prompt_usd_per_token
judge_out_cap = 0 if use_system_one_judge else JUDGE_HIGH_OUTPUT_TOKENS
judge_comp_price = max(0.0, float(judge_completion_usd_per_token or 0.0))
judge_high = (judge_calls * 1024 * judge_prompt_usd_per_token) + (
judge_calls * judge_out_cap * judge_comp_price
)
component_usd["judge"] = (judge_low, judge_high)
low_usd += judge_low
high_usd += judge_high
else:
caveats.append(
Caveat(
level="info",
code="judge_calls_not_on_visitor_key",
message=(
f"This selection makes {judge_calls} LLM-judge calls through JudgeRouter. "
f"The judge model is {judge_model_id} at {judge_api_base}. "
"Those calls are not billed to the visitor OpenRouter key, and the judge model "
"is not replaced."
),
)
)
caveats.extend(_preflight_caveats(config, protocol, selection))
blocked = any(caveat.level == "error" for caveat in caveats)
preflight_ok = not any(caveat.code.startswith("preflight_") and caveat.level == "error" for caveat in caveats)
is_full = any(caveat.code == "full_item_set" for caveat in caveats)
covers_core = _covers_core(catalog, selection)
cfg_hash = compute_plan_fingerprint(
openrouter_model=protocol.openrouter_model,
judge_openrouter_model=cleaned_judge,
use_system_one_judge=bool(use_system_one_judge),
use_multiturn_chat_model=bool(use_multiturn_chat_model),
multiturn_openrouter_model=cleaned_multiturn,
modules=selection.modules,
source_files=selection.source_files,
dimensions=selection.dimensions,
max_samples=selection.max_samples,
max_samples_per_file=selection.max_samples_per_file,
)
return Plan(
protocol=protocol,
selection=selection,
caveats=caveats,
standard_requests=len(samples),
standard_low_input_tokens=low_input,
standard_high_input_tokens=high_input,
standard_output_cap_tokens=output_cap,
judge_calls=judge_calls,
judge_model_id=judge_model_id,
judge_api_base=judge_api_base,
basic_tasks=basic_tasks,
basic_target_calls_low=basic_low,
basic_target_calls_high=basic_high,
max_tokens=max_tokens,
prompt_usd_per_token=prompt_usd_per_token if price_ok else None,
completion_usd_per_token=completion_usd_per_token if price_ok else None,
low_usd=low_usd,
high_usd=high_usd,
preflight_ok=preflight_ok and not blocked,
blocked=blocked,
needs_price_ack=not price_ok,
file_counts=sorted(file_counts.items()),
judge_model_config=judge_model_config,
use_system_one_judge=bool(use_system_one_judge),
multiturn_model_config=multiturn_model_config,
use_multiturn_chat_model=bool(use_multiturn_chat_model),
config_hash=cfg_hash,
is_full_benchmark=is_full,
covers_core=covers_core,
judge_prices=_price_tuple(judge_prompt_usd_per_token, judge_completion_usd_per_token),
student_prices=(
_price_tuple(multiturn_prompt_usd_per_token, multiturn_completion_usd_per_token)
if multiturn_model_config is not None
else None
),
component_usd=component_usd,
routes=applied_routes,
)
def _apply_routes(routes: dict[str, Route], **configs: ModelConfig | None) -> dict[str, Route]:
"""Put each role's OpenRouter ``provider`` object into its request body. Decision models are not routed."""
applied: dict[str, Route] = {}
for role, model_config in configs.items():
route = routes.get(role)
if model_config is None or route is None or not route.provider:
continue
model_config.provider_kwargs = {**model_config.provider_kwargs, "provider": dict(route.provider)}
applied[role] = route
return applied
def _covers_core(catalog: Catalog, selection: RunSelection) -> bool:
"""True when every question outside the region-specific sets is selected, uncapped and unfiltered."""
active = [
name
for name in catalog.config.app.benchmark_modules.active
if name in catalog.config.modules and catalog.config.modules[name].enabled
]
core = {entry.canonical_name for entry in catalog.files} - set(CHINA_SPECIFIC_FILES)
return (
set(selection.modules) == set(active)
and core <= set(selection.source_files)
and not selection.dimensions
and selection.max_samples is None
and selection.max_samples_per_file is None
)
def _price_tuple(prompt: float | None, completion: float | None) -> tuple[float, float] | None:
if prompt is None or prompt < 0:
return None
return float(prompt), max(0.0, float(completion or 0.0))
def iter_standard_samples(config: ProjectConfig, selection: RunSelection) -> list[Sample]:
"""Same sample sequence ``BenchmarkRunner._load_samples`` produces."""
if not selection.modules or not selection.source_files:
return []
module_selection = resolve_module_selection(config, set(selection.modules))
if not module_selection.standard_modules:
return []
resolved = FileRegistry(config).resolve(
modules=set(module_selection.standard_modules),
subsets=None,
source_files=set(selection.source_files),
)
dimensions = set(selection.dimensions) or None
runner = BenchmarkRunner(config)
return list(runner._load_samples(resolved, dimensions, selection.max_samples, selection.max_samples_per_file))
def render_plan(plan: Plan) -> str:
lines = ["### Protocol", ""]
model = plan.protocol.model_config
lines.append(f"- Model field: `{model.model_name}`")
lines.append(f"- Endpoint: `{model.api_base}` + the provider `endpoint_path`")
lines.append(
f"- temperature `{model.temperature}`, max_tokens `{model.max_tokens}`, "
f"stream `{str(bool(model.supports_stream)).lower()}`, "
f"responses API `{str(plan.protocol.responses_api).lower()}`"
)
if plan.protocol.matched_card_id:
lines.append(f"- Copied card: `{plan.protocol.matched_card_id}`")
else:
lines.append("- Copied card: none")
if plan.judge_model_config is not None:
judge_mode = "System 1 (`/systemone`)" if plan.use_system_one_judge else "OpenRouter Chat (`/chat/completions`)"
lines.append(f"- Judge endpoint: `{plan.judge_model_config.model_name}` ({judge_mode})")
if plan.multiturn_model_config is not None:
lines.append(f"- Multi-turn chat endpoint (`stu`): `{plan.multiturn_model_config.model_name}`")
lines.append("")
lines.append("### Selection")
lines.append("")
lines.append(f"- Standard requests: {plan.standard_requests}")
if plan.file_counts:
for name, count in plan.file_counts:
lines.append(f" - `{english_file(name)}`: {count}")
lines.append(
f"- Input tokens, low / high: {plan.standard_low_input_tokens} / {plan.standard_high_input_tokens}"
)
lines.append(f"- Standard output cap: {plan.standard_output_cap_tokens} tokens")
lines.append(f"- LLM-judge calls: {plan.judge_calls}")
lines.append(f"- Basic-education tasks: {plan.basic_tasks}")
lines.append(
"- Basic-education target-model calls, low / high: "
f"{plan.basic_target_calls_low} / {plan.basic_target_calls_high}"
)
if plan.config_hash:
lines.append(f"- Configuration checkpoint fingerprint: `{plan.config_hash}`")
lines.append("")
lines.append("### Cost range charged to the OpenRouter key")
lines.append("")
if plan.low_usd is None or plan.high_usd is None:
lines.append("No price is available for this model id.")
else:
lines.append(f"- Low: {_usd(plan.low_usd)}")
lines.append(f"- High: {_usd(plan.high_usd)}")
lines.append(
f"- Prices: prompt {_per_million(plan.prompt_usd_per_token)} / 1M, "
f"completion {_per_million(plan.completion_usd_per_token)} / 1M"
)
lines.append("")
lines.append("### Caveats")
lines.append("")
if not plan.caveats:
lines.append("None.")
else:
for caveat in plan.caveats:
lines.append(f"- **{caveat.level}** `{caveat.code}`: {caveat.message}")
if plan.blocked:
lines.append("")
lines.append("The run is blocked until every error above is cleared.")
return "\n".join(lines)
def _selection_caveats(catalog: Catalog, selection: RunSelection, max_concurrency: int) -> list[Caveat]:
caveats: list[Caveat] = []
known = catalog.by_name()
if not selection.modules:
caveats.append(Caveat("error", "no_modules", "Select at least one module."))
if not selection.source_files:
caveats.append(Caveat("error", "no_files", "Select at least one source file."))
unknown = sorted(name for name in selection.source_files if name not in known)
if unknown:
caveats.append(
Caveat(
"error",
"unknown_files",
"These source files are not in the registry: " + ", ".join(unknown),
)
)
if selection.max_samples is not None and selection.max_samples <= 0:
caveats.append(
Caveat(
"error",
"bad_max_samples",
"Max samples must be empty (every selected item) or a positive integer.",
)
)
if max_concurrency < 1 or max_concurrency > 100:
caveats.append(
Caveat(
"error",
"bad_concurrency",
"Max concurrency must be from 1 to 100. 100 is the cap in configs/app.yaml.",
)
)
active = [
name
for name in catalog.config.app.benchmark_modules.active
if name in catalog.config.modules and catalog.config.modules[name].enabled
]
all_files = {entry.canonical_name for entry in catalog.files}
full_modules = set(selection.modules) == set(active)
full_files = set(selection.source_files) == all_files
full_items = (
full_modules
and full_files
and not selection.dimensions
and selection.max_samples is None
and selection.max_samples_per_file is None
)
if not full_items:
reasons: list[str] = []
if not full_modules:
missing = [english_module(name) for name in active if name not in selection.modules]
reasons.append("modules omitted: " + ", ".join(missing))
if not full_files:
missing_files = [name for name in catalog.files if name.canonical_name not in selection.source_files]
reasons.append(f"{len(missing_files)} registry files omitted")
if selection.dimensions:
reasons.append("dimension filter is set")
if selection.max_samples is not None:
reasons.append(
f"max_samples={selection.max_samples} keeps a prefix in registry order, "
"the same order BenchmarkRunner uses, not a random or stratified sample"
)
if selection.max_samples_per_file is not None:
reasons.append(f"max_samples_per_file={selection.max_samples_per_file} keeps a prefix of each file")
caveats.append(
Caveat(
"warning",
"not_full_benchmark",
"This selection is not the full 2,939-item benchmark ("
+ "; ".join(reasons)
+ "). Module averages from build_summary describe only the items that ran. "
"They are not comparable to the published leaderboard, and this repository "
"does not compute that table's bootstrap interval.",
)
)
else:
caveats.append(
Caveat(
"info",
"full_item_set",
"Every active module and registry file is selected, with no dimension filter "
"and no max_samples cap. build_summary will average the full item set. "
"That average is still not the published 0–100 leaderboard: the host is OpenRouter, "
"and the bootstrap interval is not implemented in this repository.",
)
)
if selection.max_samples is not None and selection.max_samples > 0:
caveats.append(
Caveat(
"warning",
"max_samples_prefix",
"max_samples stops after that many standard samples in file-registry order. "
"When basic education is also selected, standard modules consume the budget first "
"and basic education receives the remainder. That is the runner's rule.",
)
)
orphan_files = [
name
for name in selection.source_files
if name in known and known[name].module not in selection.modules
]
if orphan_files:
caveats.append(
Caveat(
"warning",
"file_module_mismatch",
"These files are checked, but their module is not selected, so the runner will skip them: "
+ ", ".join(sorted(orphan_files)),
)
)
selected_entries = [
known[name]
for name in selection.source_files
if name in known and known[name].module in selection.modules
]
for entry in selected_entries:
if entry.scoring_mode == "rule" and entry.preflight_llm_task:
caveats.append(
Caveat(
"warning",
"rule_task_preflight_hold",
f"{english_file(entry.canonical_name)} is scored by the rule judge "
f"(configs/judges.yaml task {entry.task}, mode rule). "
"Preflight still lists this task with the LLM-judge tasks and will not "
"start the run without that judge model's API key. The dashboard keeps that check.",
)
)
_scale_caveats(caveats, selected_entries)
if max_concurrency != 100:
caveats.append(
Caveat(
"info",
"concurrency",
f"max_concurrency is {max_concurrency}. The source default in configs/app.yaml is 100. "
"Concurrency changes scheduling only. Each request is still built by OpenAICompatibleClient "
"and scored by JudgeRouter. The hosted default is 8 so a 2-vCPU Space does not turn "
"timeouts into failed samples.",
)
)
if not get_api_key(VISITOR_KEY_ENV):
caveats.append(
Caveat(
"error",
"visitor_key_missing",
"Enter an OpenRouter API key. It is read from the process environment for the run "
"and is not written to configs or key.txt.",
)
)
return caveats
def _scale_caveats(caveats: list[Caveat], entries: list[CatalogFile]) -> None:
if any(entry.module == DEFAULT_BASIC_MODULE_NAME for entry in entries):
caveats.append(
Caveat(
"warning",
"basic_education_score_scale",
"Basic-education scores are the runtime's mean of numeric rubric fields. "
"The pass threshold in configs/basic_education.yaml is 3.0. "
"Other modules' rule and LLM judges write scores on a 0–1 scale "
"(teaching-harm can also be 0.5). build_summary averages each group separately. "
"This dashboard does not rescale either group onto the published 0–100 leaderboard.",
)
)
if any(entry.scoring_mode == "placeholder" for entry in entries):
caveats.append(
Caveat(
"error",
"placeholder_judge",
"A selected task has no judge in configs/judges.yaml. The placeholder judge does not emit a score.",
)
)
def _basic_education_bounds(
config: ProjectConfig,
model_config,
selection: RunSelection,
*,
standard_loaded: int,
) -> tuple[int, int, int, list[Caveat]]:
caveats: list[Caveat] = []
if DEFAULT_BASIC_MODULE_NAME not in selection.modules:
return 0, 0, 0, caveats
if not selection.source_files:
return 0, 0, 0, caveats
executor = BasicEducationExecutor(config, model_config, _SilentLogger())
scenarios = executor._select_scenarios(
subsets=None,
source_files=set(selection.source_files),
dimensions=set(selection.dimensions) or None,
)
remaining = selection.max_samples
if remaining is not None:
remaining = max(0, remaining - standard_loaded)
if standard_loaded and DEFAULT_BASIC_MODULE_NAME in selection.modules:
caveats.append(
Caveat(
"warning",
"mixed_max_samples",
"Standard modules and basic education share one max_samples budget. "
"Standard samples are counted first. Basic education is skipped when nothing remains.",
)
)
tasks = 0
low_calls = 0
high_calls = 0
for scenario in scenarios:
if remaining is not None and remaining <= 0:
break
template = executor._read_yaml_file(scenario.template_path)
declared = count_runtime_tasks(template)
if scenario.expected_tasks is not None and declared != scenario.expected_tasks:
caveats.append(
Caveat(
"warning",
"basic_education_task_count_mismatch",
f"{scenario.scenario_id} declares {declared} runtime tasks but "
f"configs/basic_education.yaml expected_tasks is {scenario.expected_tasks}. "
"The executor logs this and runs the template.",
)
)
take = declared if remaining is None else min(declared, remaining)
if selection.max_samples_per_file is not None:
take = min(take, selection.max_samples_per_file)
globals_section = template.get("globals") if isinstance(template, dict) else {}
raw_limit = globals_section.get("recursion_limit") if isinstance(globals_section, dict) else None
try:
recursion_limit = int(raw_limit)
except (TypeError, ValueError):
recursion_limit = 25
caveats.append(
Caveat(
"info",
"recursion_limit_default",
f"{scenario.scenario_id} has no globals.recursion_limit. "
"The runtime entity default is 25.",
)
)
if scenario.multi_turn:
recursion_limit = min(recursion_limit, DEFAULT_MULTI_TURN_RECURSION_LIMIT)
tasks += take
low_calls += take
high_calls += take * max(1, recursion_limit)
if remaining is not None:
remaining -= take
if scenarios:
caveats.append(
Caveat(
"warning",
"basic_education_call_bound",
"Basic-education prompts are assembled by basic_education_runtime, not by the "
"standard prompt loader. The low call bound is one target-model call per task. "
"The high bound is one target-model call per graph step, capped by the recursion "
"limit the executor writes (20 on multi-turn scenarios, otherwise the template value). "
"Basic-education judge calls and the guided-teaching simulated student are not in the "
"dollar range; with a visitor judge they are billed to the visitor key and count toward "
"the judge and student spending limits. Input tokens for these calls are also omitted.",
)
)
caveats.append(
Caveat(
"warning",
"basic_education_key_on_disk",
"The basic-education executor writes the target API key into "
"outputs/basic_education/<run_id>/<model_id>/<scenario>/basic_education_runtime.yaml "
"because the runtime reads it from that file. The dashboard deletes that run directory "
"when the run finishes. A killed process can leave the file behind.",
)
)
return tasks, low_calls, high_calls, caveats
def _preflight_caveats(config: ProjectConfig, protocol: VisitorProtocol, selection: RunSelection) -> list[Caveat]:
if not selection.modules or not selection.source_files:
return []
if selection.max_samples is not None and selection.max_samples <= 0:
return []
report = PreflightRunner(config).run(
model_id=protocol.model_config.model_id,
modules=set(selection.modules),
subsets=None,
source_files=set(selection.source_files),
dimensions=set(selection.dimensions) or None,
max_samples=selection.max_samples,
judge_enabled=True,
require_real_judges=True,
)
caveats: list[Caveat] = []
for issue in report.issues:
caveats.append(
Caveat(
level=issue.level,
code=f"preflight_{issue.code}",
message=english_preflight_message(issue),
)
)
return caveats
def _task_mode(config: ProjectConfig, task: str | None) -> str:
if not task:
return "placeholder"
task_config = config.judges.tasks.get(task)
if task_config is None:
return "placeholder"
return task_config.mode
def _judge_endpoint(config: ProjectConfig) -> tuple[str | None, str | None]:
model_id = config.judges.default_judge_model_id
if not model_id or model_id not in config.models:
return model_id, None
return model_id, config.models[model_id].api_base
def _usd(value: float) -> str:
if value >= 0.01:
return f"${value:,.2f}"
if value == 0:
return "$0"
return f"${value:.4f}"
def _per_million(price_per_token: float | None) -> str:
if price_per_token is None:
return "n/a"
return f"${price_per_token * 1_000_000:.4f}"
class _SilentLogger:
def info(self, *args, **kwargs) -> None:
return None
def warning(self, *args, **kwargs) -> None:
return None
def error(self, *args, **kwargs) -> None:
return None
def exception(self, *args, **kwargs) -> None:
return None
|