Spaces:
Running
Running
Commit Β·
bb01fd2
1
Parent(s): f2f4974
fix cost
Browse files- app.py +1 -0
- assets/pinchbench.css +16 -0
- src/export_results.py +34 -0
- src/load_entries.py +69 -22
- src/models.py +3 -0
- src/token_display.py +154 -0
- src/traces.py +38 -35
- src/ui.py +15 -3
app.py
CHANGED
|
@@ -340,6 +340,7 @@ def build_demo(entries=None):
|
|
| 340 |
"Avg duration": entry.avg_duration_seconds,
|
| 341 |
"Avg steps": entry.avg_agent_steps,
|
| 342 |
"Tokens/task": entry.avg_tokens_per_task,
|
|
|
|
| 343 |
"Total $": entry.total_cost_usd,
|
| 344 |
"Pass@k": entry.pass_at_k,
|
| 345 |
}
|
|
|
|
| 340 |
"Avg duration": entry.avg_duration_seconds,
|
| 341 |
"Avg steps": entry.avg_agent_steps,
|
| 342 |
"Tokens/task": entry.avg_tokens_per_task,
|
| 343 |
+
"Total tokens": entry.total_tokens,
|
| 344 |
"Total $": entry.total_cost_usd,
|
| 345 |
"Pass@k": entry.pass_at_k,
|
| 346 |
}
|
assets/pinchbench.css
CHANGED
|
@@ -315,6 +315,22 @@
|
|
| 315 |
min-width: 220px;
|
| 316 |
}
|
| 317 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
.wab-model-name {
|
| 319 |
font-family: var(--mono);
|
| 320 |
font-size: 0.86rem;
|
|
|
|
| 315 |
min-width: 220px;
|
| 316 |
}
|
| 317 |
|
| 318 |
+
.wab-token-split-cell {
|
| 319 |
+
min-width: 180px;
|
| 320 |
+
max-width: 320px;
|
| 321 |
+
font-size: 0.78rem;
|
| 322 |
+
line-height: 1.35;
|
| 323 |
+
color: var(--text-muted);
|
| 324 |
+
}
|
| 325 |
+
|
| 326 |
+
.wab-token-split {
|
| 327 |
+
display: inline-block;
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
.wab-traces-placeholder {
|
| 331 |
+
min-height: 120px;
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
.wab-model-name {
|
| 335 |
font-family: var(--mono);
|
| 336 |
font-size: 0.86rem;
|
src/export_results.py
CHANGED
|
@@ -14,6 +14,7 @@ from typing import Any
|
|
| 14 |
|
| 15 |
from src.bench_config import load_bench_config
|
| 16 |
from src.models import LeaderboardEntry, SectionStats, reconcile_success_metrics
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
def _sanitize_name(value: str) -> str:
|
|
@@ -111,6 +112,9 @@ def _ui_badges(payload: dict[str, Any], limit: int = 4) -> list[str]:
|
|
| 111 |
|
| 112 |
|
| 113 |
def _cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | None]:
|
|
|
|
|
|
|
|
|
|
| 114 |
_ensure_repo_on_path()
|
| 115 |
try:
|
| 116 |
from bench_eval.llm_cost import compute_results_cost
|
|
@@ -121,6 +125,35 @@ def _cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | N
|
|
| 121 |
return None, None
|
| 122 |
|
| 123 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
def _needs_cost_backfill(entry: LeaderboardEntry) -> bool:
|
| 125 |
return entry.total_cost_usd is None or entry.total_cost_usd <= 0
|
| 126 |
|
|
@@ -212,6 +245,7 @@ def export_entry(payload: dict[str, Any], *, source_path: str | None = None) ->
|
|
| 212 |
total_tokens=run.get("total_tokens"),
|
| 213 |
total_cost_usd=total_cost,
|
| 214 |
avg_cost_per_task_usd=avg_cost,
|
|
|
|
| 215 |
pass_at_k=float(pass_overall) if isinstance(pass_overall, (int, float)) else None,
|
| 216 |
agent_completion_rate=extended.get("agent_completion_rate"),
|
| 217 |
agent_dab_agreement_rate=extended.get("agent_dab_agreement_rate"),
|
|
|
|
| 14 |
|
| 15 |
from src.bench_config import load_bench_config
|
| 16 |
from src.models import LeaderboardEntry, SectionStats, reconcile_success_metrics
|
| 17 |
+
from src.token_display import compact_token_usage_by_model, token_usage_from_run
|
| 18 |
|
| 19 |
|
| 20 |
def _sanitize_name(value: str) -> str:
|
|
|
|
| 112 |
|
| 113 |
|
| 114 |
def _cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | None]:
|
| 115 |
+
recorded = _recorded_cost_from_payload(payload)
|
| 116 |
+
if recorded is not None:
|
| 117 |
+
return recorded
|
| 118 |
_ensure_repo_on_path()
|
| 119 |
try:
|
| 120 |
from bench_eval.llm_cost import compute_results_cost
|
|
|
|
| 125 |
return None, None
|
| 126 |
|
| 127 |
|
| 128 |
+
def _recorded_cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | None] | None:
|
| 129 |
+
"""Fast path: sum recorded cost_usd without fetching pricing tables."""
|
| 130 |
+
run = payload.get("run") or {}
|
| 131 |
+
tests = payload.get("tests") or []
|
| 132 |
+
total_tasks = int(run.get("total_tasks") or len(tests) or 0)
|
| 133 |
+
by_model = run.get("token_usage_by_model")
|
| 134 |
+
if isinstance(by_model, dict) and by_model:
|
| 135 |
+
total = 0.0
|
| 136 |
+
priced = False
|
| 137 |
+
for usage in by_model.values():
|
| 138 |
+
if not isinstance(usage, dict):
|
| 139 |
+
continue
|
| 140 |
+
cost = usage.get("cost_usd")
|
| 141 |
+
if isinstance(cost, (int, float)) and float(cost) > 0:
|
| 142 |
+
total += float(cost)
|
| 143 |
+
priced = True
|
| 144 |
+
if priced:
|
| 145 |
+
avg = (total / total_tasks) if total_tasks else None
|
| 146 |
+
return total, avg
|
| 147 |
+
|
| 148 |
+
for key in ("total_cost_usd", "total_cost", "cost_usd", "cost"):
|
| 149 |
+
value = run.get(key)
|
| 150 |
+
if isinstance(value, (int, float)) and float(value) > 0:
|
| 151 |
+
total = float(value)
|
| 152 |
+
avg = (total / total_tasks) if total_tasks else None
|
| 153 |
+
return total, avg
|
| 154 |
+
return None
|
| 155 |
+
|
| 156 |
+
|
| 157 |
def _needs_cost_backfill(entry: LeaderboardEntry) -> bool:
|
| 158 |
return entry.total_cost_usd is None or entry.total_cost_usd <= 0
|
| 159 |
|
|
|
|
| 245 |
total_tokens=run.get("total_tokens"),
|
| 246 |
total_cost_usd=total_cost,
|
| 247 |
avg_cost_per_task_usd=avg_cost,
|
| 248 |
+
token_usage_by_model=token_usage_from_run(run),
|
| 249 |
pass_at_k=float(pass_overall) if isinstance(pass_overall, (int, float)) else None,
|
| 250 |
agent_completion_rate=extended.get("agent_completion_rate"),
|
| 251 |
agent_dab_agreement_rate=extended.get("agent_dab_agreement_rate"),
|
src/load_entries.py
CHANGED
|
@@ -7,8 +7,16 @@ from pathlib import Path
|
|
| 7 |
from typing import Any
|
| 8 |
|
| 9 |
from src.bench_config import load_bench_config
|
| 10 |
-
from src.export_results import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
from src.models import LeaderboardEntry
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
|
| 14 |
def space_root(root: Path | None = None) -> Path:
|
|
@@ -19,6 +27,56 @@ def results_dir(root: Path | None = None) -> Path:
|
|
| 19 |
return space_root(root) / "results"
|
| 20 |
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
def load_entry_file(path: Path) -> LeaderboardEntry | None:
|
| 23 |
payload = json.loads(path.read_text(encoding="utf-8"))
|
| 24 |
if "metrics" not in payload:
|
|
@@ -28,17 +86,18 @@ def load_entry_file(path: Path) -> LeaderboardEntry | None:
|
|
| 28 |
raw_path = raw_results_path_for(entry, space_root(path.parent.parent))
|
| 29 |
if raw_path.is_file():
|
| 30 |
entry.raw_results_path = str(raw_path.relative_to(space_root(path.parent.parent)))
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
raw = load_raw_results(entry, space_root(path.parent.parent))
|
| 37 |
if raw:
|
| 38 |
-
|
| 39 |
-
if total_cost is not None and total_cost > 0:
|
| 40 |
-
entry.total_cost_usd = total_cost
|
| 41 |
-
entry.avg_cost_per_task_usd = avg_cost
|
| 42 |
return entry
|
| 43 |
|
| 44 |
|
|
@@ -73,18 +132,6 @@ def entry_by_id(entries: list[LeaderboardEntry], entry_id: str) -> LeaderboardEn
|
|
| 73 |
return None
|
| 74 |
|
| 75 |
|
| 76 |
-
def load_raw_results(entry: LeaderboardEntry, root: Path | None = None) -> dict[str, Any] | None:
|
| 77 |
-
base = space_root(root)
|
| 78 |
-
if entry.raw_results_path:
|
| 79 |
-
raw_path = base / entry.raw_results_path
|
| 80 |
-
if raw_path.is_file():
|
| 81 |
-
return json.loads(raw_path.read_text(encoding="utf-8"))
|
| 82 |
-
fallback = raw_results_path_for(entry, base)
|
| 83 |
-
if fallback.is_file():
|
| 84 |
-
return json.loads(fallback.read_text(encoding="utf-8"))
|
| 85 |
-
return None
|
| 86 |
-
|
| 87 |
-
|
| 88 |
def sort_entries(entries: list[LeaderboardEntry], view_id: str) -> list[LeaderboardEntry]:
|
| 89 |
config = load_bench_config()
|
| 90 |
view = next((row for row in config["views"] if row["id"] == view_id), config["views"][0])
|
|
|
|
| 7 |
from typing import Any
|
| 8 |
|
| 9 |
from src.bench_config import load_bench_config
|
| 10 |
+
from src.export_results import (
|
| 11 |
+
_cost_from_payload,
|
| 12 |
+
_recorded_cost_from_payload,
|
| 13 |
+
raw_results_path_for,
|
| 14 |
+
ui_classes_from_payload,
|
| 15 |
+
)
|
| 16 |
from src.models import LeaderboardEntry
|
| 17 |
+
from src.token_display import token_usage_from_run
|
| 18 |
+
|
| 19 |
+
_RAW_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
|
| 20 |
|
| 21 |
|
| 22 |
def space_root(root: Path | None = None) -> Path:
|
|
|
|
| 27 |
return space_root(root) / "results"
|
| 28 |
|
| 29 |
|
| 30 |
+
def _resolve_raw_path(entry: LeaderboardEntry, root: Path | None = None) -> Path | None:
|
| 31 |
+
base = space_root(root)
|
| 32 |
+
if entry.raw_results_path:
|
| 33 |
+
raw_path = base / entry.raw_results_path
|
| 34 |
+
if raw_path.is_file():
|
| 35 |
+
return raw_path
|
| 36 |
+
fallback = raw_results_path_for(entry, base)
|
| 37 |
+
if fallback.is_file():
|
| 38 |
+
return fallback
|
| 39 |
+
return None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def load_raw_results(entry: LeaderboardEntry, root: Path | None = None) -> dict[str, Any] | None:
|
| 43 |
+
raw_path = _resolve_raw_path(entry, root)
|
| 44 |
+
if raw_path is None:
|
| 45 |
+
return None
|
| 46 |
+
cache_key = str(raw_path.resolve())
|
| 47 |
+
try:
|
| 48 |
+
mtime = raw_path.stat().st_mtime
|
| 49 |
+
except OSError:
|
| 50 |
+
mtime = 0.0
|
| 51 |
+
cached = _RAW_CACHE.get(cache_key)
|
| 52 |
+
if cached is not None and cached[0] == mtime:
|
| 53 |
+
return cached[1]
|
| 54 |
+
payload = json.loads(raw_path.read_text(encoding="utf-8"))
|
| 55 |
+
_RAW_CACHE[cache_key] = (mtime, payload)
|
| 56 |
+
return payload
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _enrich_entry_from_raw(entry: LeaderboardEntry, raw: dict[str, Any]) -> None:
|
| 60 |
+
run = raw.get("run") or {}
|
| 61 |
+
if not entry.ui_classes:
|
| 62 |
+
entry.ui_classes = ui_classes_from_payload(raw)
|
| 63 |
+
if not entry.token_usage_by_model:
|
| 64 |
+
entry.token_usage_by_model = token_usage_from_run(run)
|
| 65 |
+
if entry.total_cost_usd is None or entry.total_cost_usd <= 0:
|
| 66 |
+
recorded = _recorded_cost_from_payload(raw)
|
| 67 |
+
if recorded is not None:
|
| 68 |
+
total_cost, avg_cost = recorded
|
| 69 |
+
else:
|
| 70 |
+
total_cost, avg_cost = _cost_from_payload(raw)
|
| 71 |
+
if total_cost is not None and total_cost > 0:
|
| 72 |
+
entry.total_cost_usd = total_cost
|
| 73 |
+
entry.avg_cost_per_task_usd = avg_cost
|
| 74 |
+
if entry.total_tokens is None and isinstance(run.get("total_tokens"), (int, float)):
|
| 75 |
+
entry.total_tokens = int(run["total_tokens"])
|
| 76 |
+
if entry.avg_tokens_per_task is None and isinstance(run.get("avg_tokens_per_task"), (int, float)):
|
| 77 |
+
entry.avg_tokens_per_task = float(run["avg_tokens_per_task"])
|
| 78 |
+
|
| 79 |
+
|
| 80 |
def load_entry_file(path: Path) -> LeaderboardEntry | None:
|
| 81 |
payload = json.loads(path.read_text(encoding="utf-8"))
|
| 82 |
if "metrics" not in payload:
|
|
|
|
| 86 |
raw_path = raw_results_path_for(entry, space_root(path.parent.parent))
|
| 87 |
if raw_path.is_file():
|
| 88 |
entry.raw_results_path = str(raw_path.relative_to(space_root(path.parent.parent)))
|
| 89 |
+
needs_raw = (
|
| 90 |
+
not entry.ui_classes
|
| 91 |
+
or not entry.token_usage_by_model
|
| 92 |
+
or entry.total_cost_usd is None
|
| 93 |
+
or entry.total_cost_usd <= 0
|
| 94 |
+
or entry.total_tokens is None
|
| 95 |
+
or entry.avg_tokens_per_task is None
|
| 96 |
+
)
|
| 97 |
+
if needs_raw:
|
| 98 |
raw = load_raw_results(entry, space_root(path.parent.parent))
|
| 99 |
if raw:
|
| 100 |
+
_enrich_entry_from_raw(entry, raw)
|
|
|
|
|
|
|
|
|
|
| 101 |
return entry
|
| 102 |
|
| 103 |
|
|
|
|
| 132 |
return None
|
| 133 |
|
| 134 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
def sort_entries(entries: list[LeaderboardEntry], view_id: str) -> list[LeaderboardEntry]:
|
| 136 |
config = load_bench_config()
|
| 137 |
view = next((row for row in config["views"] if row["id"] == view_id), config["views"][0])
|
src/models.py
CHANGED
|
@@ -56,6 +56,7 @@ class LeaderboardEntry:
|
|
| 56 |
total_tokens: int | None = None
|
| 57 |
total_cost_usd: float | None = None
|
| 58 |
avg_cost_per_task_usd: float | None = None
|
|
|
|
| 59 |
|
| 60 |
pass_at_k: float | None = None
|
| 61 |
agent_completion_rate: float | None = None
|
|
@@ -133,6 +134,7 @@ class LeaderboardEntry:
|
|
| 133 |
"total_tokens": self.total_tokens,
|
| 134 |
"total_cost_usd": self.total_cost_usd,
|
| 135 |
"avg_cost_per_task_usd": self.avg_cost_per_task_usd,
|
|
|
|
| 136 |
"pass_at_k": self.pass_at_k,
|
| 137 |
"agent_completion_rate": self.agent_completion_rate,
|
| 138 |
"agent_dab_agreement_rate": self.agent_dab_agreement_rate,
|
|
@@ -189,6 +191,7 @@ class LeaderboardEntry:
|
|
| 189 |
total_tokens=metrics.get("total_tokens"),
|
| 190 |
total_cost_usd=metrics.get("total_cost_usd"),
|
| 191 |
avg_cost_per_task_usd=metrics.get("avg_cost_per_task_usd"),
|
|
|
|
| 192 |
pass_at_k=metrics.get("pass_at_k"),
|
| 193 |
agent_completion_rate=metrics.get("agent_completion_rate"),
|
| 194 |
agent_dab_agreement_rate=metrics.get("agent_dab_agreement_rate"),
|
|
|
|
| 56 |
total_tokens: int | None = None
|
| 57 |
total_cost_usd: float | None = None
|
| 58 |
avg_cost_per_task_usd: float | None = None
|
| 59 |
+
token_usage_by_model: dict[str, dict[str, Any]] = field(default_factory=dict)
|
| 60 |
|
| 61 |
pass_at_k: float | None = None
|
| 62 |
agent_completion_rate: float | None = None
|
|
|
|
| 134 |
"total_tokens": self.total_tokens,
|
| 135 |
"total_cost_usd": self.total_cost_usd,
|
| 136 |
"avg_cost_per_task_usd": self.avg_cost_per_task_usd,
|
| 137 |
+
"token_usage_by_model": self.token_usage_by_model,
|
| 138 |
"pass_at_k": self.pass_at_k,
|
| 139 |
"agent_completion_rate": self.agent_completion_rate,
|
| 140 |
"agent_dab_agreement_rate": self.agent_dab_agreement_rate,
|
|
|
|
| 191 |
total_tokens=metrics.get("total_tokens"),
|
| 192 |
total_cost_usd=metrics.get("total_cost_usd"),
|
| 193 |
avg_cost_per_task_usd=metrics.get("avg_cost_per_task_usd"),
|
| 194 |
+
token_usage_by_model=dict(metrics.get("token_usage_by_model") or {}),
|
| 195 |
pass_at_k=metrics.get("pass_at_k"),
|
| 196 |
agent_completion_rate=metrics.get("agent_completion_rate"),
|
| 197 |
agent_dab_agreement_rate=metrics.get("agent_dab_agreement_rate"),
|
src/token_display.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Token counts and per-model usage labels for the leaderboard."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import html
|
| 6 |
+
from typing import Any
|
| 7 |
+
|
| 8 |
+
from src.models import LeaderboardEntry
|
| 9 |
+
|
| 10 |
+
_UNKNOWN_MODEL_KEYS = frozenset({"", "unknown", "unknown_model"})
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _fmt_cost(value: float | None) -> str:
|
| 14 |
+
if value is None:
|
| 15 |
+
return "β"
|
| 16 |
+
if value <= 0:
|
| 17 |
+
return "FREE"
|
| 18 |
+
return f"${value:.3f}"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def fmt_tokens(value: float | int | None) -> str:
|
| 22 |
+
if value is None:
|
| 23 |
+
return "β"
|
| 24 |
+
amount = float(value)
|
| 25 |
+
if amount <= 0:
|
| 26 |
+
return "0"
|
| 27 |
+
if amount >= 1_000_000:
|
| 28 |
+
return f"{amount / 1_000_000:.1f}M"
|
| 29 |
+
if amount >= 1_000:
|
| 30 |
+
return f"{amount / 1_000:.1f}K"
|
| 31 |
+
if abs(amount - round(amount)) < 0.05:
|
| 32 |
+
return str(int(round(amount)))
|
| 33 |
+
return f"{amount:.1f}"
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _usage_tokens(usage: dict[str, Any]) -> int:
|
| 37 |
+
total = usage.get("total_tokens")
|
| 38 |
+
if isinstance(total, (int, float)) and total > 0:
|
| 39 |
+
return int(total)
|
| 40 |
+
prompt = int(usage.get("prompt_tokens") or 0)
|
| 41 |
+
completion = int(usage.get("completion_tokens") or 0)
|
| 42 |
+
return prompt + completion
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _display_model_name(model: str, primary_model: str) -> str:
|
| 46 |
+
normalized = str(model or "").strip()
|
| 47 |
+
if normalized.lower() in _UNKNOWN_MODEL_KEYS:
|
| 48 |
+
return primary_model
|
| 49 |
+
return normalized
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _role_label(model: str, primary_model: str, roles: list[str]) -> str:
|
| 53 |
+
display = _display_model_name(model, primary_model)
|
| 54 |
+
if display == primary_model and not roles:
|
| 55 |
+
return "primary"
|
| 56 |
+
if roles:
|
| 57 |
+
return "/".join(roles)
|
| 58 |
+
if display == primary_model:
|
| 59 |
+
return "primary"
|
| 60 |
+
return "aux"
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def compact_token_usage_by_model(
|
| 64 |
+
usage_by_model: dict[str, Any] | None,
|
| 65 |
+
) -> dict[str, dict[str, Any]]:
|
| 66 |
+
if not isinstance(usage_by_model, dict):
|
| 67 |
+
return {}
|
| 68 |
+
compact: dict[str, dict[str, Any]] = {}
|
| 69 |
+
for model, usage in usage_by_model.items():
|
| 70 |
+
if not isinstance(usage, dict):
|
| 71 |
+
continue
|
| 72 |
+
row: dict[str, Any] = {
|
| 73 |
+
"prompt_tokens": int(usage.get("prompt_tokens") or 0),
|
| 74 |
+
"completion_tokens": int(usage.get("completion_tokens") or 0),
|
| 75 |
+
"total_tokens": int(usage.get("total_tokens") or 0),
|
| 76 |
+
"llm_calls": int(usage.get("llm_calls") or 0),
|
| 77 |
+
}
|
| 78 |
+
cost = usage.get("cost_usd")
|
| 79 |
+
if isinstance(cost, (int, float)) and cost > 0:
|
| 80 |
+
row["cost_usd"] = float(cost)
|
| 81 |
+
roles = usage.get("roles")
|
| 82 |
+
if isinstance(roles, list) and roles:
|
| 83 |
+
row["roles"] = [str(role) for role in roles]
|
| 84 |
+
compact[str(model)] = row
|
| 85 |
+
return compact
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def token_usage_from_run(run: dict[str, Any] | None) -> dict[str, dict[str, Any]]:
|
| 89 |
+
if not isinstance(run, dict):
|
| 90 |
+
return {}
|
| 91 |
+
return compact_token_usage_by_model(run.get("token_usage_by_model"))
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def format_model_usage_breakdown(entry: LeaderboardEntry) -> str:
|
| 95 |
+
by_model = entry.token_usage_by_model or {}
|
| 96 |
+
if not by_model:
|
| 97 |
+
return "β"
|
| 98 |
+
|
| 99 |
+
primary = entry.model
|
| 100 |
+
rows: list[tuple[str, str, str, str]] = []
|
| 101 |
+
for model, usage in sorted(by_model.items()):
|
| 102 |
+
if not isinstance(usage, dict):
|
| 103 |
+
continue
|
| 104 |
+
display = _display_model_name(str(model), primary)
|
| 105 |
+
short = display.split("/")[-1] if "/" in display else display
|
| 106 |
+
role = _role_label(str(model), primary, list(usage.get("roles") or []))
|
| 107 |
+
tokens = fmt_tokens(_usage_tokens(usage))
|
| 108 |
+
cost = usage.get("cost_usd")
|
| 109 |
+
cost_s = _fmt_cost(float(cost)) if isinstance(cost, (int, float)) and cost > 0 else ""
|
| 110 |
+
rows.append((short, role, tokens, cost_s))
|
| 111 |
+
|
| 112 |
+
if not rows:
|
| 113 |
+
return "β"
|
| 114 |
+
|
| 115 |
+
if len(rows) == 1 and rows[0][1] == "primary":
|
| 116 |
+
short, _, tokens, cost_s = rows[0]
|
| 117 |
+
extra = f" Β· {cost_s}" if cost_s and cost_s != "β" else ""
|
| 118 |
+
return f"{short} {tokens}{extra}"
|
| 119 |
+
|
| 120 |
+
parts: list[str] = []
|
| 121 |
+
for short, role, tokens, cost_s in rows:
|
| 122 |
+
label = "primary" if role == "primary" else role
|
| 123 |
+
bit = f"{short} {tokens} ({label})"
|
| 124 |
+
if cost_s and cost_s != "β":
|
| 125 |
+
bit += f" {cost_s}"
|
| 126 |
+
parts.append(bit)
|
| 127 |
+
return " Β· ".join(parts)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def format_model_usage_breakdown_html(entry: LeaderboardEntry) -> str:
|
| 131 |
+
text = format_model_usage_breakdown(entry)
|
| 132 |
+
if text == "β":
|
| 133 |
+
return "β"
|
| 134 |
+
return f'<span class="wab-token-split" title="{html.escape(text)}">{html.escape(text)}</span>'
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def format_task_token_usage(
|
| 138 |
+
test: dict[str, Any],
|
| 139 |
+
*,
|
| 140 |
+
primary_model: str,
|
| 141 |
+
) -> str:
|
| 142 |
+
by_model = test.get("token_usage_by_model")
|
| 143 |
+
if isinstance(by_model, dict) and by_model:
|
| 144 |
+
entry = LeaderboardEntry(
|
| 145 |
+
model=primary_model,
|
| 146 |
+
harness=str(test.get("agent_harness") or ""),
|
| 147 |
+
token_usage_by_model=compact_token_usage_by_model(by_model),
|
| 148 |
+
)
|
| 149 |
+
return format_model_usage_breakdown(entry)
|
| 150 |
+
|
| 151 |
+
usage = test.get("token_usage")
|
| 152 |
+
if isinstance(usage, dict):
|
| 153 |
+
return fmt_tokens(_usage_tokens(usage))
|
| 154 |
+
return "β"
|
src/traces.py
CHANGED
|
@@ -10,6 +10,7 @@ from typing import Any
|
|
| 10 |
from src.bench_config import load_bench_config, section_label
|
| 11 |
from src.load_entries import entry_by_id, entry_label, load_raw_results
|
| 12 |
from src.models import LeaderboardEntry
|
|
|
|
| 13 |
from src.ui import fmt_cost
|
| 14 |
|
| 15 |
|
|
@@ -395,10 +396,15 @@ def build_traces_shell(
|
|
| 395 |
|
| 396 |
|
| 397 |
def _build_traces_list(entries: list[LeaderboardEntry], selected_id: str) -> str:
|
|
|
|
|
|
|
| 398 |
pills: list[str] = []
|
| 399 |
for entry in entries:
|
| 400 |
-
|
| 401 |
-
|
|
|
|
|
|
|
|
|
|
| 402 |
active = entry.entry_id == selected_id
|
| 403 |
label = html.escape(entry.model.split("/")[-1] if "/" in entry.model else entry.model)
|
| 404 |
harness = html.escape(entry.harness)
|
|
@@ -410,32 +416,30 @@ def _build_traces_list(entries: list[LeaderboardEntry], selected_id: str) -> str
|
|
| 410 |
pills.append(_trace_pill(entry.entry_id, inner, active=active))
|
| 411 |
|
| 412 |
cards: list[str] = []
|
| 413 |
-
for
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
tags = []
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
)
|
| 438 |
-
cards.append(_trace_card(entry.entry_id, test_name, card_inner, visible=visible))
|
| 439 |
|
| 440 |
body = "".join(cards) if cards else '<p class="wab-detail-note">No task traces in this run.</p>'
|
| 441 |
return (
|
|
@@ -485,6 +489,7 @@ def build_trace_detail(entries: list[LeaderboardEntry], entry_id: str, test_name
|
|
| 485 |
section = _task_section_label(test_name)
|
| 486 |
token_usage = test.get("token_usage") or agent.get("token_usage") or {}
|
| 487 |
task_cost = _task_cost_usd(test, entry.model)
|
|
|
|
| 488 |
success = bool(test.get("success"))
|
| 489 |
|
| 490 |
meta_grid = "".join(
|
|
@@ -497,6 +502,7 @@ def build_trace_detail(entries: list[LeaderboardEntry], entry_id: str, test_name
|
|
| 497 |
_meta_cell("π", "Harness", harness),
|
| 498 |
_meta_cell("π£", "Steps", test.get("agent_steps") or max(0, len(ui_steps) - 1)),
|
| 499 |
_meta_cell("πͺ", "Tokens", _format_tokens(token_usage)),
|
|
|
|
| 500 |
_meta_cell("π°", "Cost", fmt_cost(task_cost)),
|
| 501 |
_meta_cell("β±", "Duration", format_duration(test.get("duration_seconds"))),
|
| 502 |
_meta_cell("π©", "Result", "PASS" if success else "FAIL"),
|
|
@@ -697,7 +703,9 @@ element.addEventListener("click", (event) => {
|
|
| 697 |
const pill = event.target.closest("[data-wab-trace-filter]");
|
| 698 |
if (pill) {
|
| 699 |
event.preventDefault();
|
| 700 |
-
|
|
|
|
|
|
|
| 701 |
return;
|
| 702 |
}
|
| 703 |
|
|
@@ -707,12 +715,7 @@ element.addEventListener("click", (event) => {
|
|
| 707 |
const payload = navBtn.getAttribute("data-wab-trace");
|
| 708 |
if (!payload) return;
|
| 709 |
if (payload.startsWith("traces:list:")) {
|
| 710 |
-
|
| 711 |
-
if (element.querySelector("#wab-trace-detail")) {
|
| 712 |
-
trigger("navigate", { payload });
|
| 713 |
-
return;
|
| 714 |
-
}
|
| 715 |
-
filterTraceEntry(element, entryId);
|
| 716 |
return;
|
| 717 |
}
|
| 718 |
if (window.wabOpenDetailTab) window.wabOpenDetailTab("traces");
|
|
|
|
| 10 |
from src.bench_config import load_bench_config, section_label
|
| 11 |
from src.load_entries import entry_by_id, entry_label, load_raw_results
|
| 12 |
from src.models import LeaderboardEntry
|
| 13 |
+
from src.token_display import format_task_token_usage
|
| 14 |
from src.ui import fmt_cost
|
| 15 |
|
| 16 |
|
|
|
|
| 396 |
|
| 397 |
|
| 398 |
def _build_traces_list(entries: list[LeaderboardEntry], selected_id: str) -> str:
|
| 399 |
+
selected_entry = entry_by_id(entries, selected_id) or entries[0]
|
| 400 |
+
selected_summaries = trace_summaries_for_entry(selected_entry)
|
| 401 |
pills: list[str] = []
|
| 402 |
for entry in entries:
|
| 403 |
+
count = (
|
| 404 |
+
len(selected_summaries)
|
| 405 |
+
if entry.entry_id == selected_entry.entry_id
|
| 406 |
+
else int(entry.total_tasks or 0)
|
| 407 |
+
)
|
| 408 |
active = entry.entry_id == selected_id
|
| 409 |
label = html.escape(entry.model.split("/")[-1] if "/" in entry.model else entry.model)
|
| 410 |
harness = html.escape(entry.harness)
|
|
|
|
| 416 |
pills.append(_trace_pill(entry.entry_id, inner, active=active))
|
| 417 |
|
| 418 |
cards: list[str] = []
|
| 419 |
+
for row in selected_summaries:
|
| 420 |
+
test_name = row["test_name"]
|
| 421 |
+
short_name = test_name.replace("bench_", "", 1) if test_name.startswith("bench_") else test_name
|
| 422 |
+
tags = []
|
| 423 |
+
if row["domain"]:
|
| 424 |
+
tags.append(f'<span class="wab-badge domain">{_esc(row["domain"])}</span>')
|
| 425 |
+
if row["primary_class"]:
|
| 426 |
+
tags.append(f'<span class="wab-badge taxonomy">{_esc(row["primary_class"])}</span>')
|
| 427 |
+
tags_html = "".join(tags)
|
| 428 |
+
card_inner = (
|
| 429 |
+
f'<div class="wab-trace-card-accent {"pass" if row["success"] else "fail"}"></div>'
|
| 430 |
+
f'<div class="wab-trace-card-body">'
|
| 431 |
+
f'<div class="wab-trace-card-top">'
|
| 432 |
+
f'<span class="wab-trace-card-id">#{_esc(short_name)}</span>'
|
| 433 |
+
f'<span class="wab-trace-card-section">{_esc(row["section"])}</span>'
|
| 434 |
+
f"{_status_badge(row['success'])}"
|
| 435 |
+
f'<span class="wab-trace-card-duration">β± {_esc(format_duration(row["duration"]))}</span>'
|
| 436 |
+
f'<span class="wab-trace-card-steps">{row["agent_steps"]} steps Β· {row["dab_events"]} DAB</span>'
|
| 437 |
+
f"</div>"
|
| 438 |
+
f'<p class="wab-trace-card-task">{_esc(_truncate(row["input"], 220))}</p>'
|
| 439 |
+
f'<div class="wab-trace-card-tags">{tags_html}</div>'
|
| 440 |
+
f"</div>"
|
| 441 |
+
)
|
| 442 |
+
cards.append(_trace_card(selected_entry.entry_id, test_name, card_inner))
|
|
|
|
|
|
|
| 443 |
|
| 444 |
body = "".join(cards) if cards else '<p class="wab-detail-note">No task traces in this run.</p>'
|
| 445 |
return (
|
|
|
|
| 489 |
section = _task_section_label(test_name)
|
| 490 |
token_usage = test.get("token_usage") or agent.get("token_usage") or {}
|
| 491 |
task_cost = _task_cost_usd(test, entry.model)
|
| 492 |
+
model_usage = format_task_token_usage(test, primary_model=entry.model)
|
| 493 |
success = bool(test.get("success"))
|
| 494 |
|
| 495 |
meta_grid = "".join(
|
|
|
|
| 502 |
_meta_cell("π", "Harness", harness),
|
| 503 |
_meta_cell("π£", "Steps", test.get("agent_steps") or max(0, len(ui_steps) - 1)),
|
| 504 |
_meta_cell("πͺ", "Tokens", _format_tokens(token_usage)),
|
| 505 |
+
_meta_cell("π§©", "Model usage", model_usage),
|
| 506 |
_meta_cell("π°", "Cost", fmt_cost(task_cost)),
|
| 507 |
_meta_cell("β±", "Duration", format_duration(test.get("duration_seconds"))),
|
| 508 |
_meta_cell("π©", "Result", "PASS" if success else "FAIL"),
|
|
|
|
| 703 |
const pill = event.target.closest("[data-wab-trace-filter]");
|
| 704 |
if (pill) {
|
| 705 |
event.preventDefault();
|
| 706 |
+
const entryId = pill.getAttribute("data-wab-trace-filter");
|
| 707 |
+
if (!entryId) return;
|
| 708 |
+
trigger("navigate", { payload: `traces:list:${entryId}` });
|
| 709 |
return;
|
| 710 |
}
|
| 711 |
|
|
|
|
| 715 |
const payload = navBtn.getAttribute("data-wab-trace");
|
| 716 |
if (!payload) return;
|
| 717 |
if (payload.startsWith("traces:list:")) {
|
| 718 |
+
trigger("navigate", { payload });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 719 |
return;
|
| 720 |
}
|
| 721 |
if (window.wabOpenDetailTab) window.wabOpenDetailTab("traces");
|
src/ui.py
CHANGED
|
@@ -9,6 +9,7 @@ from typing import Any
|
|
| 9 |
from src.bench_config import load_bench_config, task_count
|
| 10 |
from src.load_entries import sort_entries
|
| 11 |
from src.models import LeaderboardEntry
|
|
|
|
| 12 |
|
| 13 |
_DETAIL_TAB_LABELS = {
|
| 14 |
"submission": "Submission",
|
|
@@ -266,6 +267,9 @@ def _cost_row(idx: int, entry: LeaderboardEntry) -> str:
|
|
| 266 |
<div class="wab-badges"><span class="wab-badge">{html.escape(entry.harness)}</span></div></td>
|
| 267 |
<td>{html.escape(fmt_cost(entry.total_cost_usd))}</td>
|
| 268 |
<td>{html.escape(fmt_cost(entry.avg_cost_per_task_usd))}</td>
|
|
|
|
|
|
|
|
|
|
| 269 |
<td>{f"{value:.1f}" if value is not None else "β"}</td>
|
| 270 |
<td class="wab-score-cell">
|
| 271 |
<div class="wab-score-value wab-score-{tier}">{html.escape(fmt_pct_display(entry))}</div>
|
|
@@ -285,9 +289,17 @@ def build_table(entries: list[LeaderboardEntry], view_id: str) -> str:
|
|
| 285 |
rows = [_speed_row(i, entry) for i, entry in enumerate(sorted_entries)]
|
| 286 |
subtitle = "Average task duration from results.json run.avg_duration_seconds."
|
| 287 |
elif view_id == "cost":
|
| 288 |
-
head =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
rows = [_cost_row(i, entry) for i, entry in enumerate(sorted_entries)]
|
| 290 |
-
subtitle =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
else:
|
| 292 |
head = "<tr><th>#</th><th>Model</th><th>Harness</th><th>Tasks</th><th>Success %</th></tr>"
|
| 293 |
rows = [_success_row(i, entry) for i, entry in enumerate(sorted_entries)]
|
|
@@ -296,7 +308,7 @@ def build_table(entries: list[LeaderboardEntry], view_id: str) -> str:
|
|
| 296 |
"See docs/EVAL_README.md."
|
| 297 |
)
|
| 298 |
|
| 299 |
-
body = "".join(rows) if rows else f'<tr><td colspan="
|
| 300 |
return f"""
|
| 301 |
<div class="wab-panel">
|
| 302 |
<div class="wab-panel-head">
|
|
|
|
| 9 |
from src.bench_config import load_bench_config, task_count
|
| 10 |
from src.load_entries import sort_entries
|
| 11 |
from src.models import LeaderboardEntry
|
| 12 |
+
from src.token_display import fmt_tokens, format_model_usage_breakdown_html
|
| 13 |
|
| 14 |
_DETAIL_TAB_LABELS = {
|
| 15 |
"submission": "Submission",
|
|
|
|
| 267 |
<div class="wab-badges"><span class="wab-badge">{html.escape(entry.harness)}</span></div></td>
|
| 268 |
<td>{html.escape(fmt_cost(entry.total_cost_usd))}</td>
|
| 269 |
<td>{html.escape(fmt_cost(entry.avg_cost_per_task_usd))}</td>
|
| 270 |
+
<td>{html.escape(fmt_tokens(entry.total_tokens))}</td>
|
| 271 |
+
<td>{html.escape(fmt_tokens(entry.avg_tokens_per_task))}</td>
|
| 272 |
+
<td class="wab-token-split-cell">{format_model_usage_breakdown_html(entry)}</td>
|
| 273 |
<td>{f"{value:.1f}" if value is not None else "β"}</td>
|
| 274 |
<td class="wab-score-cell">
|
| 275 |
<div class="wab-score-value wab-score-{tier}">{html.escape(fmt_pct_display(entry))}</div>
|
|
|
|
| 289 |
rows = [_speed_row(i, entry) for i, entry in enumerate(sorted_entries)]
|
| 290 |
subtitle = "Average task duration from results.json run.avg_duration_seconds."
|
| 291 |
elif view_id == "cost":
|
| 292 |
+
head = (
|
| 293 |
+
"<tr><th>#</th><th>Model</th><th>Total $</th><th>$/task</th>"
|
| 294 |
+
"<th>Total tokens</th><th>Tokens/task</th><th>Model usage</th>"
|
| 295 |
+
"<th>Value</th><th>Success %</th></tr>"
|
| 296 |
+
)
|
| 297 |
rows = [_cost_row(i, entry) for i, entry in enumerate(sorted_entries)]
|
| 298 |
+
subtitle = (
|
| 299 |
+
"LLM cost via bench_eval.llm_cost. Tokens from run totals. "
|
| 300 |
+
"Model usage splits primary run model vs Ouroboros auxiliary slots (review, fallback, β¦). "
|
| 301 |
+
"Value = success_rate / total_cost_usd."
|
| 302 |
+
)
|
| 303 |
else:
|
| 304 |
head = "<tr><th>#</th><th>Model</th><th>Harness</th><th>Tasks</th><th>Success %</th></tr>"
|
| 305 |
rows = [_success_row(i, entry) for i, entry in enumerate(sorted_entries)]
|
|
|
|
| 308 |
"See docs/EVAL_README.md."
|
| 309 |
)
|
| 310 |
|
| 311 |
+
body = "".join(rows) if rows else f'<tr><td colspan="9">No results yet. Run export_results.py.</td></tr>'
|
| 312 |
return f"""
|
| 313 |
<div class="wab-panel">
|
| 314 |
<div class="wab-panel-head">
|