ai-forever commited on
Commit
fd4a73d
·
1 Parent(s): dbb27c0

fix vision

Browse files
assets/pinchbench.css CHANGED
@@ -347,6 +347,16 @@
347
  white-space: nowrap;
348
  }
349
 
 
 
 
 
 
 
 
 
 
 
350
  .wab-traces-placeholder {
351
  min-height: 120px;
352
  }
 
347
  white-space: nowrap;
348
  }
349
 
350
+ .wab-token-split-total {
351
+ display: block;
352
+ margin-top: 0.15rem;
353
+ overflow: hidden;
354
+ text-overflow: ellipsis;
355
+ white-space: nowrap;
356
+ color: var(--text);
357
+ font-weight: 600;
358
+ }
359
+
360
  .wab-traces-placeholder {
361
  min-height: 120px;
362
  }
src/load_entries.py CHANGED
@@ -14,7 +14,7 @@ from src.export_results import (
14
  ui_classes_from_payload,
15
  )
16
  from src.models import LeaderboardEntry
17
- from src.token_display import token_usage_from_run
18
 
19
  _RAW_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
20
 
@@ -82,6 +82,7 @@ def _enrich_entry_from_raw(entry: LeaderboardEntry, raw: dict[str, Any]) -> None
82
  for slot, models in slots.items()
83
  if isinstance(models, list) and models
84
  }
 
85
 
86
 
87
  def load_entry_file(path: Path) -> LeaderboardEntry | None:
@@ -106,6 +107,8 @@ def load_entry_file(path: Path) -> LeaderboardEntry | None:
106
  raw = load_raw_results(entry, space_root(path.parent.parent))
107
  if raw:
108
  _enrich_entry_from_raw(entry, raw)
 
 
109
  return entry
110
 
111
 
 
14
  ui_classes_from_payload,
15
  )
16
  from src.models import LeaderboardEntry
17
+ from src.token_display import reconcile_entry_usage_totals, token_usage_from_run
18
 
19
  _RAW_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
20
 
 
82
  for slot, models in slots.items()
83
  if isinstance(models, list) and models
84
  }
85
+ reconcile_entry_usage_totals(entry)
86
 
87
 
88
  def load_entry_file(path: Path) -> LeaderboardEntry | None:
 
107
  raw = load_raw_results(entry, space_root(path.parent.parent))
108
  if raw:
109
  _enrich_entry_from_raw(entry, raw)
110
+ else:
111
+ reconcile_entry_usage_totals(entry)
112
  return entry
113
 
114
 
src/token_display.py CHANGED
@@ -85,6 +85,46 @@ def token_usage_from_run(run: dict[str, Any] | None) -> dict[str, dict[str, Any]
85
  return compact_token_usage_by_model(run.get("token_usage_by_model"))
86
 
87
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  def _short_model_name(model: str) -> str:
89
  return model.split("/")[-1] if "/" in model else model
90
 
@@ -162,17 +202,33 @@ def _compact_slot_label(model: str, roles: list[str], *, primary: str) -> str:
162
  return short
163
 
164
 
165
- def _ouroboros_slot_lines(entry: LeaderboardEntry) -> list[str]:
166
  index = _model_roles_index(entry.ouroboros_model_slots)
167
  if not index:
168
  return []
 
 
 
 
 
 
 
 
 
169
  if len(index) == 1:
170
- return [_short_model_name(next(iter(index)))]
 
 
 
 
171
  primary = entry.model
172
- return [
173
  _compact_slot_label(model, index[model], primary=primary)
174
  for model in sorted(index, key=lambda name: (name != primary, name))
175
  ]
 
 
 
176
 
177
 
178
  def _format_token_usage_breakdown(entry: LeaderboardEntry) -> str:
@@ -209,7 +265,7 @@ def _format_token_usage_breakdown(entry: LeaderboardEntry) -> str:
209
 
210
 
211
  def _format_ouroboros_slots_breakdown(entry: LeaderboardEntry) -> str:
212
- lines = _ouroboros_slot_lines(entry)
213
  if not lines:
214
  return "—"
215
  return " · ".join(lines)
@@ -230,13 +286,15 @@ def format_model_usage_breakdown_html(entry: LeaderboardEntry) -> str:
230
  text = _format_token_usage_breakdown(entry)
231
  return f'<span class="wab-token-split" title="{html.escape(text)}">{html.escape(text)}</span>'
232
 
233
- lines = _ouroboros_slot_lines(entry)
234
  if not lines:
235
  return "—"
236
  title = " · ".join(lines)
237
- body = "".join(
238
- f'<span class="wab-token-split-line">{html.escape(line)}</span>' for line in lines
239
- )
 
 
240
  return f'<span class="wab-token-split" title="{html.escape(title)}">{body}</span>'
241
 
242
 
@@ -266,6 +324,7 @@ def format_task_token_usage(
266
  entry = LeaderboardEntry(
267
  model=primary_model,
268
  harness=harness,
 
269
  ouroboros_model_slots=slots,
270
  )
271
  if _has_ouroboros_slot_split(entry):
 
85
  return compact_token_usage_by_model(run.get("token_usage_by_model"))
86
 
87
 
88
+ def sum_usage_by_model(
89
+ usage_by_model: dict[str, dict[str, Any]] | None,
90
+ ) -> tuple[int, float | None]:
91
+ """Sum token and recorded cost counters across per-model buckets."""
92
+ tokens = 0
93
+ cost = 0.0
94
+ has_cost = False
95
+ for usage in (usage_by_model or {}).values():
96
+ if not isinstance(usage, dict):
97
+ continue
98
+ tokens += _usage_tokens(usage)
99
+ amount = usage.get("cost_usd")
100
+ if isinstance(amount, (int, float)) and float(amount) > 0:
101
+ cost += float(amount)
102
+ has_cost = True
103
+ return tokens, (cost if has_cost else None)
104
+
105
+
106
+ def entry_recorded_totals(entry: LeaderboardEntry) -> tuple[int | None, float | None]:
107
+ """Return authoritative totals from per-model buckets, falling back to run totals."""
108
+ tokens, cost = sum_usage_by_model(entry.token_usage_by_model)
109
+ total_tokens = tokens if tokens > 0 else entry.total_tokens
110
+ total_cost = cost if cost is not None and cost > 0 else entry.total_cost_usd
111
+ return total_tokens, total_cost
112
+
113
+
114
+ def reconcile_entry_usage_totals(entry: LeaderboardEntry) -> None:
115
+ """Keep run totals aligned with summed token_usage_by_model when recorded."""
116
+ by_model = entry.token_usage_by_model or {}
117
+ if not by_model:
118
+ return
119
+ tokens, cost = sum_usage_by_model(by_model)
120
+ if tokens > 0:
121
+ entry.total_tokens = tokens
122
+ if cost is not None and cost > 0:
123
+ entry.total_cost_usd = cost
124
+ if entry.total_tasks:
125
+ entry.avg_cost_per_task_usd = cost / entry.total_tasks
126
+
127
+
128
  def _short_model_name(model: str) -> str:
129
  return model.split("/")[-1] if "/" in model else model
130
 
 
202
  return short
203
 
204
 
205
+ def _ouroboros_slot_display_lines(entry: LeaderboardEntry) -> list[str]:
206
  index = _model_roles_index(entry.ouroboros_model_slots)
207
  if not index:
208
  return []
209
+
210
+ total_tokens, total_cost = entry_recorded_totals(entry)
211
+ total_bits: list[str] = []
212
+ if total_tokens:
213
+ total_bits.append(fmt_tokens(total_tokens))
214
+ if total_cost:
215
+ total_bits.append(fmt_cost(total_cost))
216
+ total_line = " · ".join(total_bits)
217
+
218
  if len(index) == 1:
219
+ short = _short_model_name(next(iter(index)))
220
+ if total_line:
221
+ return [f"{short} {total_line}"]
222
+ return [short]
223
+
224
  primary = entry.model
225
+ lines = [
226
  _compact_slot_label(model, index[model], primary=primary)
227
  for model in sorted(index, key=lambda name: (name != primary, name))
228
  ]
229
+ if total_line:
230
+ lines.append(f"{total_line} total")
231
+ return lines
232
 
233
 
234
  def _format_token_usage_breakdown(entry: LeaderboardEntry) -> str:
 
265
 
266
 
267
  def _format_ouroboros_slots_breakdown(entry: LeaderboardEntry) -> str:
268
+ lines = _ouroboros_slot_display_lines(entry)
269
  if not lines:
270
  return "—"
271
  return " · ".join(lines)
 
286
  text = _format_token_usage_breakdown(entry)
287
  return f'<span class="wab-token-split" title="{html.escape(text)}">{html.escape(text)}</span>'
288
 
289
+ lines = _ouroboros_slot_display_lines(entry)
290
  if not lines:
291
  return "—"
292
  title = " · ".join(lines)
293
+ body_parts: list[str] = []
294
+ for line in lines:
295
+ css_class = "wab-token-split-total" if line.endswith(" total") else "wab-token-split-line"
296
+ body_parts.append(f'<span class="{css_class}">{html.escape(line)}</span>')
297
+ body = "".join(body_parts)
298
  return f'<span class="wab-token-split" title="{html.escape(title)}">{body}</span>'
299
 
300
 
 
324
  entry = LeaderboardEntry(
325
  model=primary_model,
326
  harness=harness,
327
+ token_usage_by_model=compact_token_usage_by_model(by_model if isinstance(by_model, dict) else None),
328
  ouroboros_model_slots=slots,
329
  )
330
  if _has_ouroboros_slot_split(entry):