ai-forever commited on
Commit
bb01fd2
Β·
1 Parent(s): f2f4974
app.py CHANGED
@@ -340,6 +340,7 @@ def build_demo(entries=None):
340
  "Avg duration": entry.avg_duration_seconds,
341
  "Avg steps": entry.avg_agent_steps,
342
  "Tokens/task": entry.avg_tokens_per_task,
 
343
  "Total $": entry.total_cost_usd,
344
  "Pass@k": entry.pass_at_k,
345
  }
 
340
  "Avg duration": entry.avg_duration_seconds,
341
  "Avg steps": entry.avg_agent_steps,
342
  "Tokens/task": entry.avg_tokens_per_task,
343
+ "Total tokens": entry.total_tokens,
344
  "Total $": entry.total_cost_usd,
345
  "Pass@k": entry.pass_at_k,
346
  }
assets/pinchbench.css CHANGED
@@ -315,6 +315,22 @@
315
  min-width: 220px;
316
  }
317
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318
  .wab-model-name {
319
  font-family: var(--mono);
320
  font-size: 0.86rem;
 
315
  min-width: 220px;
316
  }
317
 
318
+ .wab-token-split-cell {
319
+ min-width: 180px;
320
+ max-width: 320px;
321
+ font-size: 0.78rem;
322
+ line-height: 1.35;
323
+ color: var(--text-muted);
324
+ }
325
+
326
+ .wab-token-split {
327
+ display: inline-block;
328
+ }
329
+
330
+ .wab-traces-placeholder {
331
+ min-height: 120px;
332
+ }
333
+
334
  .wab-model-name {
335
  font-family: var(--mono);
336
  font-size: 0.86rem;
src/export_results.py CHANGED
@@ -14,6 +14,7 @@ from typing import Any
14
 
15
  from src.bench_config import load_bench_config
16
  from src.models import LeaderboardEntry, SectionStats, reconcile_success_metrics
 
17
 
18
 
19
  def _sanitize_name(value: str) -> str:
@@ -111,6 +112,9 @@ def _ui_badges(payload: dict[str, Any], limit: int = 4) -> list[str]:
111
 
112
 
113
  def _cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | None]:
 
 
 
114
  _ensure_repo_on_path()
115
  try:
116
  from bench_eval.llm_cost import compute_results_cost
@@ -121,6 +125,35 @@ def _cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | N
121
  return None, None
122
 
123
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  def _needs_cost_backfill(entry: LeaderboardEntry) -> bool:
125
  return entry.total_cost_usd is None or entry.total_cost_usd <= 0
126
 
@@ -212,6 +245,7 @@ def export_entry(payload: dict[str, Any], *, source_path: str | None = None) ->
212
  total_tokens=run.get("total_tokens"),
213
  total_cost_usd=total_cost,
214
  avg_cost_per_task_usd=avg_cost,
 
215
  pass_at_k=float(pass_overall) if isinstance(pass_overall, (int, float)) else None,
216
  agent_completion_rate=extended.get("agent_completion_rate"),
217
  agent_dab_agreement_rate=extended.get("agent_dab_agreement_rate"),
 
14
 
15
  from src.bench_config import load_bench_config
16
  from src.models import LeaderboardEntry, SectionStats, reconcile_success_metrics
17
+ from src.token_display import compact_token_usage_by_model, token_usage_from_run
18
 
19
 
20
  def _sanitize_name(value: str) -> str:
 
112
 
113
 
114
  def _cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | None]:
115
+ recorded = _recorded_cost_from_payload(payload)
116
+ if recorded is not None:
117
+ return recorded
118
  _ensure_repo_on_path()
119
  try:
120
  from bench_eval.llm_cost import compute_results_cost
 
125
  return None, None
126
 
127
 
128
+ def _recorded_cost_from_payload(payload: dict[str, Any]) -> tuple[float | None, float | None] | None:
129
+ """Fast path: sum recorded cost_usd without fetching pricing tables."""
130
+ run = payload.get("run") or {}
131
+ tests = payload.get("tests") or []
132
+ total_tasks = int(run.get("total_tasks") or len(tests) or 0)
133
+ by_model = run.get("token_usage_by_model")
134
+ if isinstance(by_model, dict) and by_model:
135
+ total = 0.0
136
+ priced = False
137
+ for usage in by_model.values():
138
+ if not isinstance(usage, dict):
139
+ continue
140
+ cost = usage.get("cost_usd")
141
+ if isinstance(cost, (int, float)) and float(cost) > 0:
142
+ total += float(cost)
143
+ priced = True
144
+ if priced:
145
+ avg = (total / total_tasks) if total_tasks else None
146
+ return total, avg
147
+
148
+ for key in ("total_cost_usd", "total_cost", "cost_usd", "cost"):
149
+ value = run.get(key)
150
+ if isinstance(value, (int, float)) and float(value) > 0:
151
+ total = float(value)
152
+ avg = (total / total_tasks) if total_tasks else None
153
+ return total, avg
154
+ return None
155
+
156
+
157
  def _needs_cost_backfill(entry: LeaderboardEntry) -> bool:
158
  return entry.total_cost_usd is None or entry.total_cost_usd <= 0
159
 
 
245
  total_tokens=run.get("total_tokens"),
246
  total_cost_usd=total_cost,
247
  avg_cost_per_task_usd=avg_cost,
248
+ token_usage_by_model=token_usage_from_run(run),
249
  pass_at_k=float(pass_overall) if isinstance(pass_overall, (int, float)) else None,
250
  agent_completion_rate=extended.get("agent_completion_rate"),
251
  agent_dab_agreement_rate=extended.get("agent_dab_agreement_rate"),
src/load_entries.py CHANGED
@@ -7,8 +7,16 @@ from pathlib import Path
7
  from typing import Any
8
 
9
  from src.bench_config import load_bench_config
10
- from src.export_results import _cost_from_payload, raw_results_path_for, ui_classes_from_payload
 
 
 
 
 
11
  from src.models import LeaderboardEntry
 
 
 
12
 
13
 
14
  def space_root(root: Path | None = None) -> Path:
@@ -19,6 +27,56 @@ def results_dir(root: Path | None = None) -> Path:
19
  return space_root(root) / "results"
20
 
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  def load_entry_file(path: Path) -> LeaderboardEntry | None:
23
  payload = json.loads(path.read_text(encoding="utf-8"))
24
  if "metrics" not in payload:
@@ -28,17 +86,18 @@ def load_entry_file(path: Path) -> LeaderboardEntry | None:
28
  raw_path = raw_results_path_for(entry, space_root(path.parent.parent))
29
  if raw_path.is_file():
30
  entry.raw_results_path = str(raw_path.relative_to(space_root(path.parent.parent)))
31
- if not entry.ui_classes:
32
- raw = load_raw_results(entry, space_root(path.parent.parent))
33
- if raw:
34
- entry.ui_classes = ui_classes_from_payload(raw)
35
- if entry.total_cost_usd is None or entry.total_cost_usd <= 0:
 
 
 
 
36
  raw = load_raw_results(entry, space_root(path.parent.parent))
37
  if raw:
38
- total_cost, avg_cost = _cost_from_payload(raw)
39
- if total_cost is not None and total_cost > 0:
40
- entry.total_cost_usd = total_cost
41
- entry.avg_cost_per_task_usd = avg_cost
42
  return entry
43
 
44
 
@@ -73,18 +132,6 @@ def entry_by_id(entries: list[LeaderboardEntry], entry_id: str) -> LeaderboardEn
73
  return None
74
 
75
 
76
- def load_raw_results(entry: LeaderboardEntry, root: Path | None = None) -> dict[str, Any] | None:
77
- base = space_root(root)
78
- if entry.raw_results_path:
79
- raw_path = base / entry.raw_results_path
80
- if raw_path.is_file():
81
- return json.loads(raw_path.read_text(encoding="utf-8"))
82
- fallback = raw_results_path_for(entry, base)
83
- if fallback.is_file():
84
- return json.loads(fallback.read_text(encoding="utf-8"))
85
- return None
86
-
87
-
88
  def sort_entries(entries: list[LeaderboardEntry], view_id: str) -> list[LeaderboardEntry]:
89
  config = load_bench_config()
90
  view = next((row for row in config["views"] if row["id"] == view_id), config["views"][0])
 
7
  from typing import Any
8
 
9
  from src.bench_config import load_bench_config
10
+ from src.export_results import (
11
+ _cost_from_payload,
12
+ _recorded_cost_from_payload,
13
+ raw_results_path_for,
14
+ ui_classes_from_payload,
15
+ )
16
  from src.models import LeaderboardEntry
17
+ from src.token_display import token_usage_from_run
18
+
19
+ _RAW_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
20
 
21
 
22
  def space_root(root: Path | None = None) -> Path:
 
27
  return space_root(root) / "results"
28
 
29
 
30
+ def _resolve_raw_path(entry: LeaderboardEntry, root: Path | None = None) -> Path | None:
31
+ base = space_root(root)
32
+ if entry.raw_results_path:
33
+ raw_path = base / entry.raw_results_path
34
+ if raw_path.is_file():
35
+ return raw_path
36
+ fallback = raw_results_path_for(entry, base)
37
+ if fallback.is_file():
38
+ return fallback
39
+ return None
40
+
41
+
42
+ def load_raw_results(entry: LeaderboardEntry, root: Path | None = None) -> dict[str, Any] | None:
43
+ raw_path = _resolve_raw_path(entry, root)
44
+ if raw_path is None:
45
+ return None
46
+ cache_key = str(raw_path.resolve())
47
+ try:
48
+ mtime = raw_path.stat().st_mtime
49
+ except OSError:
50
+ mtime = 0.0
51
+ cached = _RAW_CACHE.get(cache_key)
52
+ if cached is not None and cached[0] == mtime:
53
+ return cached[1]
54
+ payload = json.loads(raw_path.read_text(encoding="utf-8"))
55
+ _RAW_CACHE[cache_key] = (mtime, payload)
56
+ return payload
57
+
58
+
59
+ def _enrich_entry_from_raw(entry: LeaderboardEntry, raw: dict[str, Any]) -> None:
60
+ run = raw.get("run") or {}
61
+ if not entry.ui_classes:
62
+ entry.ui_classes = ui_classes_from_payload(raw)
63
+ if not entry.token_usage_by_model:
64
+ entry.token_usage_by_model = token_usage_from_run(run)
65
+ if entry.total_cost_usd is None or entry.total_cost_usd <= 0:
66
+ recorded = _recorded_cost_from_payload(raw)
67
+ if recorded is not None:
68
+ total_cost, avg_cost = recorded
69
+ else:
70
+ total_cost, avg_cost = _cost_from_payload(raw)
71
+ if total_cost is not None and total_cost > 0:
72
+ entry.total_cost_usd = total_cost
73
+ entry.avg_cost_per_task_usd = avg_cost
74
+ if entry.total_tokens is None and isinstance(run.get("total_tokens"), (int, float)):
75
+ entry.total_tokens = int(run["total_tokens"])
76
+ if entry.avg_tokens_per_task is None and isinstance(run.get("avg_tokens_per_task"), (int, float)):
77
+ entry.avg_tokens_per_task = float(run["avg_tokens_per_task"])
78
+
79
+
80
  def load_entry_file(path: Path) -> LeaderboardEntry | None:
81
  payload = json.loads(path.read_text(encoding="utf-8"))
82
  if "metrics" not in payload:
 
86
  raw_path = raw_results_path_for(entry, space_root(path.parent.parent))
87
  if raw_path.is_file():
88
  entry.raw_results_path = str(raw_path.relative_to(space_root(path.parent.parent)))
89
+ needs_raw = (
90
+ not entry.ui_classes
91
+ or not entry.token_usage_by_model
92
+ or entry.total_cost_usd is None
93
+ or entry.total_cost_usd <= 0
94
+ or entry.total_tokens is None
95
+ or entry.avg_tokens_per_task is None
96
+ )
97
+ if needs_raw:
98
  raw = load_raw_results(entry, space_root(path.parent.parent))
99
  if raw:
100
+ _enrich_entry_from_raw(entry, raw)
 
 
 
101
  return entry
102
 
103
 
 
132
  return None
133
 
134
 
 
 
 
 
 
 
 
 
 
 
 
 
135
  def sort_entries(entries: list[LeaderboardEntry], view_id: str) -> list[LeaderboardEntry]:
136
  config = load_bench_config()
137
  view = next((row for row in config["views"] if row["id"] == view_id), config["views"][0])
src/models.py CHANGED
@@ -56,6 +56,7 @@ class LeaderboardEntry:
56
  total_tokens: int | None = None
57
  total_cost_usd: float | None = None
58
  avg_cost_per_task_usd: float | None = None
 
59
 
60
  pass_at_k: float | None = None
61
  agent_completion_rate: float | None = None
@@ -133,6 +134,7 @@ class LeaderboardEntry:
133
  "total_tokens": self.total_tokens,
134
  "total_cost_usd": self.total_cost_usd,
135
  "avg_cost_per_task_usd": self.avg_cost_per_task_usd,
 
136
  "pass_at_k": self.pass_at_k,
137
  "agent_completion_rate": self.agent_completion_rate,
138
  "agent_dab_agreement_rate": self.agent_dab_agreement_rate,
@@ -189,6 +191,7 @@ class LeaderboardEntry:
189
  total_tokens=metrics.get("total_tokens"),
190
  total_cost_usd=metrics.get("total_cost_usd"),
191
  avg_cost_per_task_usd=metrics.get("avg_cost_per_task_usd"),
 
192
  pass_at_k=metrics.get("pass_at_k"),
193
  agent_completion_rate=metrics.get("agent_completion_rate"),
194
  agent_dab_agreement_rate=metrics.get("agent_dab_agreement_rate"),
 
56
  total_tokens: int | None = None
57
  total_cost_usd: float | None = None
58
  avg_cost_per_task_usd: float | None = None
59
+ token_usage_by_model: dict[str, dict[str, Any]] = field(default_factory=dict)
60
 
61
  pass_at_k: float | None = None
62
  agent_completion_rate: float | None = None
 
134
  "total_tokens": self.total_tokens,
135
  "total_cost_usd": self.total_cost_usd,
136
  "avg_cost_per_task_usd": self.avg_cost_per_task_usd,
137
+ "token_usage_by_model": self.token_usage_by_model,
138
  "pass_at_k": self.pass_at_k,
139
  "agent_completion_rate": self.agent_completion_rate,
140
  "agent_dab_agreement_rate": self.agent_dab_agreement_rate,
 
191
  total_tokens=metrics.get("total_tokens"),
192
  total_cost_usd=metrics.get("total_cost_usd"),
193
  avg_cost_per_task_usd=metrics.get("avg_cost_per_task_usd"),
194
+ token_usage_by_model=dict(metrics.get("token_usage_by_model") or {}),
195
  pass_at_k=metrics.get("pass_at_k"),
196
  agent_completion_rate=metrics.get("agent_completion_rate"),
197
  agent_dab_agreement_rate=metrics.get("agent_dab_agreement_rate"),
src/token_display.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Token counts and per-model usage labels for the leaderboard."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import html
6
+ from typing import Any
7
+
8
+ from src.models import LeaderboardEntry
9
+
10
+ _UNKNOWN_MODEL_KEYS = frozenset({"", "unknown", "unknown_model"})
11
+
12
+
13
+ def _fmt_cost(value: float | None) -> str:
14
+ if value is None:
15
+ return "β€”"
16
+ if value <= 0:
17
+ return "FREE"
18
+ return f"${value:.3f}"
19
+
20
+
21
+ def fmt_tokens(value: float | int | None) -> str:
22
+ if value is None:
23
+ return "β€”"
24
+ amount = float(value)
25
+ if amount <= 0:
26
+ return "0"
27
+ if amount >= 1_000_000:
28
+ return f"{amount / 1_000_000:.1f}M"
29
+ if amount >= 1_000:
30
+ return f"{amount / 1_000:.1f}K"
31
+ if abs(amount - round(amount)) < 0.05:
32
+ return str(int(round(amount)))
33
+ return f"{amount:.1f}"
34
+
35
+
36
+ def _usage_tokens(usage: dict[str, Any]) -> int:
37
+ total = usage.get("total_tokens")
38
+ if isinstance(total, (int, float)) and total > 0:
39
+ return int(total)
40
+ prompt = int(usage.get("prompt_tokens") or 0)
41
+ completion = int(usage.get("completion_tokens") or 0)
42
+ return prompt + completion
43
+
44
+
45
+ def _display_model_name(model: str, primary_model: str) -> str:
46
+ normalized = str(model or "").strip()
47
+ if normalized.lower() in _UNKNOWN_MODEL_KEYS:
48
+ return primary_model
49
+ return normalized
50
+
51
+
52
+ def _role_label(model: str, primary_model: str, roles: list[str]) -> str:
53
+ display = _display_model_name(model, primary_model)
54
+ if display == primary_model and not roles:
55
+ return "primary"
56
+ if roles:
57
+ return "/".join(roles)
58
+ if display == primary_model:
59
+ return "primary"
60
+ return "aux"
61
+
62
+
63
+ def compact_token_usage_by_model(
64
+ usage_by_model: dict[str, Any] | None,
65
+ ) -> dict[str, dict[str, Any]]:
66
+ if not isinstance(usage_by_model, dict):
67
+ return {}
68
+ compact: dict[str, dict[str, Any]] = {}
69
+ for model, usage in usage_by_model.items():
70
+ if not isinstance(usage, dict):
71
+ continue
72
+ row: dict[str, Any] = {
73
+ "prompt_tokens": int(usage.get("prompt_tokens") or 0),
74
+ "completion_tokens": int(usage.get("completion_tokens") or 0),
75
+ "total_tokens": int(usage.get("total_tokens") or 0),
76
+ "llm_calls": int(usage.get("llm_calls") or 0),
77
+ }
78
+ cost = usage.get("cost_usd")
79
+ if isinstance(cost, (int, float)) and cost > 0:
80
+ row["cost_usd"] = float(cost)
81
+ roles = usage.get("roles")
82
+ if isinstance(roles, list) and roles:
83
+ row["roles"] = [str(role) for role in roles]
84
+ compact[str(model)] = row
85
+ return compact
86
+
87
+
88
+ def token_usage_from_run(run: dict[str, Any] | None) -> dict[str, dict[str, Any]]:
89
+ if not isinstance(run, dict):
90
+ return {}
91
+ return compact_token_usage_by_model(run.get("token_usage_by_model"))
92
+
93
+
94
+ def format_model_usage_breakdown(entry: LeaderboardEntry) -> str:
95
+ by_model = entry.token_usage_by_model or {}
96
+ if not by_model:
97
+ return "β€”"
98
+
99
+ primary = entry.model
100
+ rows: list[tuple[str, str, str, str]] = []
101
+ for model, usage in sorted(by_model.items()):
102
+ if not isinstance(usage, dict):
103
+ continue
104
+ display = _display_model_name(str(model), primary)
105
+ short = display.split("/")[-1] if "/" in display else display
106
+ role = _role_label(str(model), primary, list(usage.get("roles") or []))
107
+ tokens = fmt_tokens(_usage_tokens(usage))
108
+ cost = usage.get("cost_usd")
109
+ cost_s = _fmt_cost(float(cost)) if isinstance(cost, (int, float)) and cost > 0 else ""
110
+ rows.append((short, role, tokens, cost_s))
111
+
112
+ if not rows:
113
+ return "β€”"
114
+
115
+ if len(rows) == 1 and rows[0][1] == "primary":
116
+ short, _, tokens, cost_s = rows[0]
117
+ extra = f" Β· {cost_s}" if cost_s and cost_s != "β€”" else ""
118
+ return f"{short} {tokens}{extra}"
119
+
120
+ parts: list[str] = []
121
+ for short, role, tokens, cost_s in rows:
122
+ label = "primary" if role == "primary" else role
123
+ bit = f"{short} {tokens} ({label})"
124
+ if cost_s and cost_s != "β€”":
125
+ bit += f" {cost_s}"
126
+ parts.append(bit)
127
+ return " Β· ".join(parts)
128
+
129
+
130
+ def format_model_usage_breakdown_html(entry: LeaderboardEntry) -> str:
131
+ text = format_model_usage_breakdown(entry)
132
+ if text == "β€”":
133
+ return "β€”"
134
+ return f'<span class="wab-token-split" title="{html.escape(text)}">{html.escape(text)}</span>'
135
+
136
+
137
+ def format_task_token_usage(
138
+ test: dict[str, Any],
139
+ *,
140
+ primary_model: str,
141
+ ) -> str:
142
+ by_model = test.get("token_usage_by_model")
143
+ if isinstance(by_model, dict) and by_model:
144
+ entry = LeaderboardEntry(
145
+ model=primary_model,
146
+ harness=str(test.get("agent_harness") or ""),
147
+ token_usage_by_model=compact_token_usage_by_model(by_model),
148
+ )
149
+ return format_model_usage_breakdown(entry)
150
+
151
+ usage = test.get("token_usage")
152
+ if isinstance(usage, dict):
153
+ return fmt_tokens(_usage_tokens(usage))
154
+ return "β€”"
src/traces.py CHANGED
@@ -10,6 +10,7 @@ from typing import Any
10
  from src.bench_config import load_bench_config, section_label
11
  from src.load_entries import entry_by_id, entry_label, load_raw_results
12
  from src.models import LeaderboardEntry
 
13
  from src.ui import fmt_cost
14
 
15
 
@@ -395,10 +396,15 @@ def build_traces_shell(
395
 
396
 
397
  def _build_traces_list(entries: list[LeaderboardEntry], selected_id: str) -> str:
 
 
398
  pills: list[str] = []
399
  for entry in entries:
400
- summaries = trace_summaries_for_entry(entry)
401
- count = len(summaries)
 
 
 
402
  active = entry.entry_id == selected_id
403
  label = html.escape(entry.model.split("/")[-1] if "/" in entry.model else entry.model)
404
  harness = html.escape(entry.harness)
@@ -410,32 +416,30 @@ def _build_traces_list(entries: list[LeaderboardEntry], selected_id: str) -> str
410
  pills.append(_trace_pill(entry.entry_id, inner, active=active))
411
 
412
  cards: list[str] = []
413
- for entry in entries:
414
- visible = entry.entry_id == selected_id
415
- for row in trace_summaries_for_entry(entry):
416
- test_name = row["test_name"]
417
- short_name = test_name.replace("bench_", "", 1) if test_name.startswith("bench_") else test_name
418
- tags = []
419
- if row["domain"]:
420
- tags.append(f'<span class="wab-badge domain">{_esc(row["domain"])}</span>')
421
- if row["primary_class"]:
422
- tags.append(f'<span class="wab-badge taxonomy">{_esc(row["primary_class"])}</span>')
423
- tags_html = "".join(tags)
424
- card_inner = (
425
- f'<div class="wab-trace-card-accent {"pass" if row["success"] else "fail"}"></div>'
426
- f'<div class="wab-trace-card-body">'
427
- f'<div class="wab-trace-card-top">'
428
- f'<span class="wab-trace-card-id">#{_esc(short_name)}</span>'
429
- f'<span class="wab-trace-card-section">{_esc(row["section"])}</span>'
430
- f"{_status_badge(row['success'])}"
431
- f'<span class="wab-trace-card-duration">⏱ {_esc(format_duration(row["duration"]))}</span>'
432
- f'<span class="wab-trace-card-steps">{row["agent_steps"]} steps Β· {row["dab_events"]} DAB</span>'
433
- f"</div>"
434
- f'<p class="wab-trace-card-task">{_esc(_truncate(row["input"], 220))}</p>'
435
- f'<div class="wab-trace-card-tags">{tags_html}</div>'
436
- f"</div>"
437
- )
438
- cards.append(_trace_card(entry.entry_id, test_name, card_inner, visible=visible))
439
 
440
  body = "".join(cards) if cards else '<p class="wab-detail-note">No task traces in this run.</p>'
441
  return (
@@ -485,6 +489,7 @@ def build_trace_detail(entries: list[LeaderboardEntry], entry_id: str, test_name
485
  section = _task_section_label(test_name)
486
  token_usage = test.get("token_usage") or agent.get("token_usage") or {}
487
  task_cost = _task_cost_usd(test, entry.model)
 
488
  success = bool(test.get("success"))
489
 
490
  meta_grid = "".join(
@@ -497,6 +502,7 @@ def build_trace_detail(entries: list[LeaderboardEntry], entry_id: str, test_name
497
  _meta_cell("🌐", "Harness", harness),
498
  _meta_cell("πŸ‘£", "Steps", test.get("agent_steps") or max(0, len(ui_steps) - 1)),
499
  _meta_cell("πŸͺ™", "Tokens", _format_tokens(token_usage)),
 
500
  _meta_cell("πŸ’°", "Cost", fmt_cost(task_cost)),
501
  _meta_cell("⏱", "Duration", format_duration(test.get("duration_seconds"))),
502
  _meta_cell("🚩", "Result", "PASS" if success else "FAIL"),
@@ -697,7 +703,9 @@ element.addEventListener("click", (event) => {
697
  const pill = event.target.closest("[data-wab-trace-filter]");
698
  if (pill) {
699
  event.preventDefault();
700
- filterTraceEntry(element, pill.getAttribute("data-wab-trace-filter"));
 
 
701
  return;
702
  }
703
 
@@ -707,12 +715,7 @@ element.addEventListener("click", (event) => {
707
  const payload = navBtn.getAttribute("data-wab-trace");
708
  if (!payload) return;
709
  if (payload.startsWith("traces:list:")) {
710
- const entryId = payload.split(":").slice(2).join(":");
711
- if (element.querySelector("#wab-trace-detail")) {
712
- trigger("navigate", { payload });
713
- return;
714
- }
715
- filterTraceEntry(element, entryId);
716
  return;
717
  }
718
  if (window.wabOpenDetailTab) window.wabOpenDetailTab("traces");
 
10
  from src.bench_config import load_bench_config, section_label
11
  from src.load_entries import entry_by_id, entry_label, load_raw_results
12
  from src.models import LeaderboardEntry
13
+ from src.token_display import format_task_token_usage
14
  from src.ui import fmt_cost
15
 
16
 
 
396
 
397
 
398
  def _build_traces_list(entries: list[LeaderboardEntry], selected_id: str) -> str:
399
+ selected_entry = entry_by_id(entries, selected_id) or entries[0]
400
+ selected_summaries = trace_summaries_for_entry(selected_entry)
401
  pills: list[str] = []
402
  for entry in entries:
403
+ count = (
404
+ len(selected_summaries)
405
+ if entry.entry_id == selected_entry.entry_id
406
+ else int(entry.total_tasks or 0)
407
+ )
408
  active = entry.entry_id == selected_id
409
  label = html.escape(entry.model.split("/")[-1] if "/" in entry.model else entry.model)
410
  harness = html.escape(entry.harness)
 
416
  pills.append(_trace_pill(entry.entry_id, inner, active=active))
417
 
418
  cards: list[str] = []
419
+ for row in selected_summaries:
420
+ test_name = row["test_name"]
421
+ short_name = test_name.replace("bench_", "", 1) if test_name.startswith("bench_") else test_name
422
+ tags = []
423
+ if row["domain"]:
424
+ tags.append(f'<span class="wab-badge domain">{_esc(row["domain"])}</span>')
425
+ if row["primary_class"]:
426
+ tags.append(f'<span class="wab-badge taxonomy">{_esc(row["primary_class"])}</span>')
427
+ tags_html = "".join(tags)
428
+ card_inner = (
429
+ f'<div class="wab-trace-card-accent {"pass" if row["success"] else "fail"}"></div>'
430
+ f'<div class="wab-trace-card-body">'
431
+ f'<div class="wab-trace-card-top">'
432
+ f'<span class="wab-trace-card-id">#{_esc(short_name)}</span>'
433
+ f'<span class="wab-trace-card-section">{_esc(row["section"])}</span>'
434
+ f"{_status_badge(row['success'])}"
435
+ f'<span class="wab-trace-card-duration">⏱ {_esc(format_duration(row["duration"]))}</span>'
436
+ f'<span class="wab-trace-card-steps">{row["agent_steps"]} steps Β· {row["dab_events"]} DAB</span>'
437
+ f"</div>"
438
+ f'<p class="wab-trace-card-task">{_esc(_truncate(row["input"], 220))}</p>'
439
+ f'<div class="wab-trace-card-tags">{tags_html}</div>'
440
+ f"</div>"
441
+ )
442
+ cards.append(_trace_card(selected_entry.entry_id, test_name, card_inner))
 
 
443
 
444
  body = "".join(cards) if cards else '<p class="wab-detail-note">No task traces in this run.</p>'
445
  return (
 
489
  section = _task_section_label(test_name)
490
  token_usage = test.get("token_usage") or agent.get("token_usage") or {}
491
  task_cost = _task_cost_usd(test, entry.model)
492
+ model_usage = format_task_token_usage(test, primary_model=entry.model)
493
  success = bool(test.get("success"))
494
 
495
  meta_grid = "".join(
 
502
  _meta_cell("🌐", "Harness", harness),
503
  _meta_cell("πŸ‘£", "Steps", test.get("agent_steps") or max(0, len(ui_steps) - 1)),
504
  _meta_cell("πŸͺ™", "Tokens", _format_tokens(token_usage)),
505
+ _meta_cell("🧩", "Model usage", model_usage),
506
  _meta_cell("πŸ’°", "Cost", fmt_cost(task_cost)),
507
  _meta_cell("⏱", "Duration", format_duration(test.get("duration_seconds"))),
508
  _meta_cell("🚩", "Result", "PASS" if success else "FAIL"),
 
703
  const pill = event.target.closest("[data-wab-trace-filter]");
704
  if (pill) {
705
  event.preventDefault();
706
+ const entryId = pill.getAttribute("data-wab-trace-filter");
707
+ if (!entryId) return;
708
+ trigger("navigate", { payload: `traces:list:${entryId}` });
709
  return;
710
  }
711
 
 
715
  const payload = navBtn.getAttribute("data-wab-trace");
716
  if (!payload) return;
717
  if (payload.startsWith("traces:list:")) {
718
+ trigger("navigate", { payload });
 
 
 
 
 
719
  return;
720
  }
721
  if (window.wabOpenDetailTab) window.wabOpenDetailTab("traces");
src/ui.py CHANGED
@@ -9,6 +9,7 @@ from typing import Any
9
  from src.bench_config import load_bench_config, task_count
10
  from src.load_entries import sort_entries
11
  from src.models import LeaderboardEntry
 
12
 
13
  _DETAIL_TAB_LABELS = {
14
  "submission": "Submission",
@@ -266,6 +267,9 @@ def _cost_row(idx: int, entry: LeaderboardEntry) -> str:
266
  <div class="wab-badges"><span class="wab-badge">{html.escape(entry.harness)}</span></div></td>
267
  <td>{html.escape(fmt_cost(entry.total_cost_usd))}</td>
268
  <td>{html.escape(fmt_cost(entry.avg_cost_per_task_usd))}</td>
 
 
 
269
  <td>{f"{value:.1f}" if value is not None else "β€”"}</td>
270
  <td class="wab-score-cell">
271
  <div class="wab-score-value wab-score-{tier}">{html.escape(fmt_pct_display(entry))}</div>
@@ -285,9 +289,17 @@ def build_table(entries: list[LeaderboardEntry], view_id: str) -> str:
285
  rows = [_speed_row(i, entry) for i, entry in enumerate(sorted_entries)]
286
  subtitle = "Average task duration from results.json run.avg_duration_seconds."
287
  elif view_id == "cost":
288
- head = "<tr><th>#</th><th>Model</th><th>Total $</th><th>$/task</th><th>Value</th><th>Success %</th></tr>"
 
 
 
 
289
  rows = [_cost_row(i, entry) for i, entry in enumerate(sorted_entries)]
290
- subtitle = "LLM cost estimated via bench_eval.llm_cost. Value = success_rate / total_cost_usd."
 
 
 
 
291
  else:
292
  head = "<tr><th>#</th><th>Model</th><th>Harness</th><th>Tasks</th><th>Success %</th></tr>"
293
  rows = [_success_row(i, entry) for i, entry in enumerate(sorted_entries)]
@@ -296,7 +308,7 @@ def build_table(entries: list[LeaderboardEntry], view_id: str) -> str:
296
  "See docs/EVAL_README.md."
297
  )
298
 
299
- body = "".join(rows) if rows else f'<tr><td colspan="6">No results yet. Run export_results.py.</td></tr>'
300
  return f"""
301
  <div class="wab-panel">
302
  <div class="wab-panel-head">
 
9
  from src.bench_config import load_bench_config, task_count
10
  from src.load_entries import sort_entries
11
  from src.models import LeaderboardEntry
12
+ from src.token_display import fmt_tokens, format_model_usage_breakdown_html
13
 
14
  _DETAIL_TAB_LABELS = {
15
  "submission": "Submission",
 
267
  <div class="wab-badges"><span class="wab-badge">{html.escape(entry.harness)}</span></div></td>
268
  <td>{html.escape(fmt_cost(entry.total_cost_usd))}</td>
269
  <td>{html.escape(fmt_cost(entry.avg_cost_per_task_usd))}</td>
270
+ <td>{html.escape(fmt_tokens(entry.total_tokens))}</td>
271
+ <td>{html.escape(fmt_tokens(entry.avg_tokens_per_task))}</td>
272
+ <td class="wab-token-split-cell">{format_model_usage_breakdown_html(entry)}</td>
273
  <td>{f"{value:.1f}" if value is not None else "β€”"}</td>
274
  <td class="wab-score-cell">
275
  <div class="wab-score-value wab-score-{tier}">{html.escape(fmt_pct_display(entry))}</div>
 
289
  rows = [_speed_row(i, entry) for i, entry in enumerate(sorted_entries)]
290
  subtitle = "Average task duration from results.json run.avg_duration_seconds."
291
  elif view_id == "cost":
292
+ head = (
293
+ "<tr><th>#</th><th>Model</th><th>Total $</th><th>$/task</th>"
294
+ "<th>Total tokens</th><th>Tokens/task</th><th>Model usage</th>"
295
+ "<th>Value</th><th>Success %</th></tr>"
296
+ )
297
  rows = [_cost_row(i, entry) for i, entry in enumerate(sorted_entries)]
298
+ subtitle = (
299
+ "LLM cost via bench_eval.llm_cost. Tokens from run totals. "
300
+ "Model usage splits primary run model vs Ouroboros auxiliary slots (review, fallback, …). "
301
+ "Value = success_rate / total_cost_usd."
302
+ )
303
  else:
304
  head = "<tr><th>#</th><th>Model</th><th>Harness</th><th>Tasks</th><th>Success %</th></tr>"
305
  rows = [_success_row(i, entry) for i, entry in enumerate(sorted_entries)]
 
308
  "See docs/EVAL_README.md."
309
  )
310
 
311
+ body = "".join(rows) if rows else f'<tr><td colspan="9">No results yet. Run export_results.py.</td></tr>'
312
  return f"""
313
  <div class="wab-panel">
314
  <div class="wab-panel-head">