"""Token counts and per-model usage labels for the leaderboard."""
from __future__ import annotations
import html
from typing import Any
from src.formatting import fmt_cost
from src.models import LeaderboardEntry
_UNKNOWN_MODEL_KEYS = frozenset({"", "unknown", "unknown_model"})
_SINGLE_MODEL_HARNESSES = frozenset({"browser-use", "ouroboros-cut"})
_OUROBOROS_FULL_PREFIX = "ouroboros-full"
_OUROBOROS_AUX_ROLES = frozenset({"fallback", "review"})
def fmt_tokens(value: float | int | None) -> str:
if value is None:
return "—"
amount = float(value)
if amount <= 0:
return "0"
if amount >= 1_000_000:
return f"{amount / 1_000_000:.2f}M"
if amount >= 1_000:
return f"{amount / 1_000:.2f}K"
return f"{amount:.2f}"
def _usage_tokens(usage: dict[str, Any]) -> int:
total = usage.get("total_tokens")
if isinstance(total, (int, float)) and total > 0:
return int(total)
prompt = int(usage.get("prompt_tokens") or 0)
completion = int(usage.get("completion_tokens") or 0)
return prompt + completion
def _display_model_name(model: str, primary_model: str) -> str:
normalized = str(model or "").strip()
if normalized.lower() in _UNKNOWN_MODEL_KEYS:
return primary_model
return normalized
def _role_label(model: str, primary_model: str, roles: list[str]) -> str:
display = _display_model_name(model, primary_model)
if display == primary_model and not roles:
return "primary"
if roles:
return "/".join(roles)
if display == primary_model:
return "primary"
return "aux"
def compact_token_usage_by_model(
usage_by_model: dict[str, Any] | None,
) -> dict[str, dict[str, Any]]:
if not isinstance(usage_by_model, dict):
return {}
compact: dict[str, dict[str, Any]] = {}
for model, usage in usage_by_model.items():
if not isinstance(usage, dict):
continue
row: dict[str, Any] = {
"prompt_tokens": int(usage.get("prompt_tokens") or 0),
"completion_tokens": int(usage.get("completion_tokens") or 0),
"total_tokens": int(usage.get("total_tokens") or 0),
"llm_calls": int(usage.get("llm_calls") or 0),
}
cost = usage.get("cost_usd")
if isinstance(cost, (int, float)) and cost > 0:
row["cost_usd"] = float(cost)
roles = usage.get("roles")
if isinstance(roles, list) and roles:
row["roles"] = [str(role) for role in roles]
compact[str(model)] = row
return compact
def token_usage_from_run(run: dict[str, Any] | None) -> dict[str, dict[str, Any]]:
if not isinstance(run, dict):
return {}
return compact_token_usage_by_model(run.get("token_usage_by_model"))
def sum_usage_by_model(
usage_by_model: dict[str, dict[str, Any]] | None,
) -> tuple[int, float | None]:
"""Sum token and recorded cost counters across per-model buckets."""
tokens = 0
cost = 0.0
has_cost = False
for usage in (usage_by_model or {}).values():
if not isinstance(usage, dict):
continue
tokens += _usage_tokens(usage)
amount = usage.get("cost_usd")
if isinstance(amount, (int, float)) and float(amount) > 0:
cost += float(amount)
has_cost = True
return tokens, (cost if has_cost else None)
def entry_recorded_totals(entry: LeaderboardEntry) -> tuple[int | None, float | None]:
"""Return authoritative totals from per-model buckets, falling back to run totals."""
tokens, cost = sum_usage_by_model(entry.token_usage_by_model)
total_tokens = tokens if tokens > 0 else entry.total_tokens
total_cost = cost if cost is not None and cost > 0 else entry.total_cost_usd
return total_tokens, total_cost
def reconcile_entry_usage_totals(entry: LeaderboardEntry) -> None:
"""Keep run totals aligned with summed token_usage_by_model when recorded."""
by_model = entry.token_usage_by_model or {}
if not by_model:
return
tokens, cost = sum_usage_by_model(by_model)
if tokens > 0:
entry.total_tokens = tokens
if cost is not None and cost > 0:
entry.total_cost_usd = cost
if entry.total_tasks:
entry.avg_cost_per_task_usd = cost / entry.total_tasks
def _short_model_name(model: str) -> str:
return model.split("/")[-1] if "/" in model else model
def _model_roles_index(model_slots: dict[str, list[str]]) -> dict[str, list[str]]:
index: dict[str, set[str]] = {}
for role, models in model_slots.items():
for model in models:
model_name = str(model or "").strip()
if not model_name:
continue
index.setdefault(model_name, set()).add(str(role))
return {model: sorted(roles) for model, roles in sorted(index.items())}
def _normalize_model_slots(slots: dict[str, Any] | None) -> dict[str, list[str]]:
if not isinstance(slots, dict):
return {}
normalized: dict[str, list[str]] = {}
for slot, models in slots.items():
if not isinstance(models, list):
continue
cleaned = [str(model) for model in models if str(model or "").strip()]
if cleaned:
normalized[str(slot)] = cleaned
return normalized
def _task_ouroboros_model_slots(test: dict[str, Any]) -> dict[str, list[str]]:
agent = test.get("agent") or {}
meta = agent.get("harness_metadata") or {}
sync = meta.get("ouroboros_llm_sync") or {}
return _normalize_model_slots(sync.get("model_slots"))
def _has_per_model_token_split(entry: LeaderboardEntry) -> bool:
by_model = entry.token_usage_by_model or {}
if not by_model:
return False
primary = entry.model
display_models: set[str] = set()
for model, usage in by_model.items():
if not isinstance(usage, dict):
continue
display_models.add(_display_model_name(str(model), primary))
roles = [str(role) for role in (usage.get("roles") or [])]
if _role_label(str(model), primary, roles) != "primary":
return True
return len(display_models) > 1
def _has_ouroboros_slot_split(entry: LeaderboardEntry) -> bool:
if not entry.harness.startswith(_OUROBOROS_FULL_PREFIX):
return False
return bool(entry.ouroboros_model_slots)
def has_model_usage_split(entry: LeaderboardEntry) -> bool:
"""True when per-model breakdown adds info beyond the run's primary LLM."""
if entry.harness in _SINGLE_MODEL_HARNESSES:
return False
if _has_per_model_token_split(entry):
return True
return _has_ouroboros_slot_split(entry)
def _compact_slot_label(model: str, roles: list[str], *, primary: str) -> str:
short = _short_model_name(model)
aux_roles = [role for role in roles if role in _OUROBOROS_AUX_ROLES]
if aux_roles:
return f"{short} ({'/'.join(aux_roles)})"
if model == primary:
return f"{short} (main)"
return short
def _ouroboros_slot_display_lines(entry: LeaderboardEntry) -> list[str]:
index = _model_roles_index(entry.ouroboros_model_slots)
if not index:
return []
total_tokens, total_cost = entry_recorded_totals(entry)
total_bits: list[str] = []
if total_tokens:
total_bits.append(fmt_tokens(total_tokens))
if total_cost:
total_bits.append(fmt_cost(total_cost))
total_line = " · ".join(total_bits)
if len(index) == 1:
short = _short_model_name(next(iter(index)))
if total_line:
return [f"{short} {total_line}"]
return [short]
primary = entry.model
lines = [
_compact_slot_label(model, index[model], primary=primary)
for model in sorted(index, key=lambda name: (name != primary, name))
]
if total_line:
lines.append(f"{total_line} total")
return lines
def _format_token_usage_breakdown(entry: LeaderboardEntry) -> str:
by_model = entry.token_usage_by_model or {}
primary = entry.model
rows: list[tuple[str, str, str, str]] = []
for model, usage in sorted(by_model.items()):
if not isinstance(usage, dict):
continue
display = _display_model_name(str(model), primary)
short = _short_model_name(display)
role = _role_label(str(model), primary, list(usage.get("roles") or []))
tokens = fmt_tokens(_usage_tokens(usage))
cost = usage.get("cost_usd")
cost_s = fmt_cost(float(cost)) if isinstance(cost, (int, float)) and cost > 0 else ""
rows.append((short, role, tokens, cost_s))
if not rows:
return "—"
if len(rows) == 1 and rows[0][1] == "primary":
short, _, tokens, cost_s = rows[0]
extra = f" · {cost_s}" if cost_s and cost_s != "—" else ""
return f"{short} {tokens}{extra}"
parts: list[str] = []
for short, role, tokens, cost_s in rows:
label = "primary" if role == "primary" else role
bit = f"{short} {tokens} ({label})"
if cost_s and cost_s != "—":
bit += f" {cost_s}"
parts.append(bit)
return " · ".join(parts)
def _format_ouroboros_slots_breakdown(entry: LeaderboardEntry) -> str:
lines = _ouroboros_slot_display_lines(entry)
if not lines:
return "—"
return " · ".join(lines)
def format_model_usage_breakdown(entry: LeaderboardEntry) -> str:
if not has_model_usage_split(entry):
return "—"
if _has_per_model_token_split(entry):
return _format_token_usage_breakdown(entry)
return _format_ouroboros_slots_breakdown(entry)
def format_model_usage_breakdown_html(entry: LeaderboardEntry) -> str:
if not has_model_usage_split(entry):
return "—"
if _has_per_model_token_split(entry):
text = _format_token_usage_breakdown(entry)
return f'{html.escape(text)}'
lines = _ouroboros_slot_display_lines(entry)
if not lines:
return "—"
title = " · ".join(lines)
body_parts: list[str] = []
for line in lines:
css_class = "wab-token-split-total" if line.endswith(" total") else "wab-token-split-line"
body_parts.append(f'{html.escape(line)}')
body = "".join(body_parts)
return f'{body}'
def format_task_token_usage(
test: dict[str, Any],
*,
primary_model: str,
ouroboros_model_slots: dict[str, list[str]] | None = None,
) -> str:
harness = str(test.get("agent_harness") or "")
if harness in _SINGLE_MODEL_HARNESSES:
return "—"
slots = _task_ouroboros_model_slots(test) or _normalize_model_slots(ouroboros_model_slots)
by_model = test.get("token_usage_by_model")
if isinstance(by_model, dict) and by_model:
entry = LeaderboardEntry(
model=primary_model,
harness=harness,
token_usage_by_model=compact_token_usage_by_model(by_model),
ouroboros_model_slots=slots,
)
if _has_per_model_token_split(entry):
return _format_token_usage_breakdown(entry)
if harness.startswith(_OUROBOROS_FULL_PREFIX) and slots:
entry = LeaderboardEntry(
model=primary_model,
harness=harness,
token_usage_by_model=compact_token_usage_by_model(by_model if isinstance(by_model, dict) else None),
ouroboros_model_slots=slots,
)
if _has_ouroboros_slot_split(entry):
return _format_ouroboros_slots_breakdown(entry)
usage = test.get("token_usage")
if isinstance(usage, dict):
return fmt_tokens(_usage_tokens(usage))
return "—"