Spaces:
Running
Running
Commit ·
fd4a73d
1
Parent(s): dbb27c0
fix vision
Browse files- assets/pinchbench.css +10 -0
- src/load_entries.py +4 -1
- src/token_display.py +67 -8
assets/pinchbench.css
CHANGED
|
@@ -347,6 +347,16 @@
|
|
| 347 |
white-space: nowrap;
|
| 348 |
}
|
| 349 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
.wab-traces-placeholder {
|
| 351 |
min-height: 120px;
|
| 352 |
}
|
|
|
|
| 347 |
white-space: nowrap;
|
| 348 |
}
|
| 349 |
|
| 350 |
+
.wab-token-split-total {
|
| 351 |
+
display: block;
|
| 352 |
+
margin-top: 0.15rem;
|
| 353 |
+
overflow: hidden;
|
| 354 |
+
text-overflow: ellipsis;
|
| 355 |
+
white-space: nowrap;
|
| 356 |
+
color: var(--text);
|
| 357 |
+
font-weight: 600;
|
| 358 |
+
}
|
| 359 |
+
|
| 360 |
.wab-traces-placeholder {
|
| 361 |
min-height: 120px;
|
| 362 |
}
|
src/load_entries.py
CHANGED
|
@@ -14,7 +14,7 @@ from src.export_results import (
|
|
| 14 |
ui_classes_from_payload,
|
| 15 |
)
|
| 16 |
from src.models import LeaderboardEntry
|
| 17 |
-
from src.token_display import token_usage_from_run
|
| 18 |
|
| 19 |
_RAW_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
|
| 20 |
|
|
@@ -82,6 +82,7 @@ def _enrich_entry_from_raw(entry: LeaderboardEntry, raw: dict[str, Any]) -> None
|
|
| 82 |
for slot, models in slots.items()
|
| 83 |
if isinstance(models, list) and models
|
| 84 |
}
|
|
|
|
| 85 |
|
| 86 |
|
| 87 |
def load_entry_file(path: Path) -> LeaderboardEntry | None:
|
|
@@ -106,6 +107,8 @@ def load_entry_file(path: Path) -> LeaderboardEntry | None:
|
|
| 106 |
raw = load_raw_results(entry, space_root(path.parent.parent))
|
| 107 |
if raw:
|
| 108 |
_enrich_entry_from_raw(entry, raw)
|
|
|
|
|
|
|
| 109 |
return entry
|
| 110 |
|
| 111 |
|
|
|
|
| 14 |
ui_classes_from_payload,
|
| 15 |
)
|
| 16 |
from src.models import LeaderboardEntry
|
| 17 |
+
from src.token_display import reconcile_entry_usage_totals, token_usage_from_run
|
| 18 |
|
| 19 |
_RAW_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
|
| 20 |
|
|
|
|
| 82 |
for slot, models in slots.items()
|
| 83 |
if isinstance(models, list) and models
|
| 84 |
}
|
| 85 |
+
reconcile_entry_usage_totals(entry)
|
| 86 |
|
| 87 |
|
| 88 |
def load_entry_file(path: Path) -> LeaderboardEntry | None:
|
|
|
|
| 107 |
raw = load_raw_results(entry, space_root(path.parent.parent))
|
| 108 |
if raw:
|
| 109 |
_enrich_entry_from_raw(entry, raw)
|
| 110 |
+
else:
|
| 111 |
+
reconcile_entry_usage_totals(entry)
|
| 112 |
return entry
|
| 113 |
|
| 114 |
|
src/token_display.py
CHANGED
|
@@ -85,6 +85,46 @@ def token_usage_from_run(run: dict[str, Any] | None) -> dict[str, dict[str, Any]
|
|
| 85 |
return compact_token_usage_by_model(run.get("token_usage_by_model"))
|
| 86 |
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
def _short_model_name(model: str) -> str:
|
| 89 |
return model.split("/")[-1] if "/" in model else model
|
| 90 |
|
|
@@ -162,17 +202,33 @@ def _compact_slot_label(model: str, roles: list[str], *, primary: str) -> str:
|
|
| 162 |
return short
|
| 163 |
|
| 164 |
|
| 165 |
-
def
|
| 166 |
index = _model_roles_index(entry.ouroboros_model_slots)
|
| 167 |
if not index:
|
| 168 |
return []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
if len(index) == 1:
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
primary = entry.model
|
| 172 |
-
|
| 173 |
_compact_slot_label(model, index[model], primary=primary)
|
| 174 |
for model in sorted(index, key=lambda name: (name != primary, name))
|
| 175 |
]
|
|
|
|
|
|
|
|
|
|
| 176 |
|
| 177 |
|
| 178 |
def _format_token_usage_breakdown(entry: LeaderboardEntry) -> str:
|
|
@@ -209,7 +265,7 @@ def _format_token_usage_breakdown(entry: LeaderboardEntry) -> str:
|
|
| 209 |
|
| 210 |
|
| 211 |
def _format_ouroboros_slots_breakdown(entry: LeaderboardEntry) -> str:
|
| 212 |
-
lines =
|
| 213 |
if not lines:
|
| 214 |
return "—"
|
| 215 |
return " · ".join(lines)
|
|
@@ -230,13 +286,15 @@ def format_model_usage_breakdown_html(entry: LeaderboardEntry) -> str:
|
|
| 230 |
text = _format_token_usage_breakdown(entry)
|
| 231 |
return f'<span class="wab-token-split" title="{html.escape(text)}">{html.escape(text)}</span>'
|
| 232 |
|
| 233 |
-
lines =
|
| 234 |
if not lines:
|
| 235 |
return "—"
|
| 236 |
title = " · ".join(lines)
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
|
|
|
|
|
|
| 240 |
return f'<span class="wab-token-split" title="{html.escape(title)}">{body}</span>'
|
| 241 |
|
| 242 |
|
|
@@ -266,6 +324,7 @@ def format_task_token_usage(
|
|
| 266 |
entry = LeaderboardEntry(
|
| 267 |
model=primary_model,
|
| 268 |
harness=harness,
|
|
|
|
| 269 |
ouroboros_model_slots=slots,
|
| 270 |
)
|
| 271 |
if _has_ouroboros_slot_split(entry):
|
|
|
|
| 85 |
return compact_token_usage_by_model(run.get("token_usage_by_model"))
|
| 86 |
|
| 87 |
|
| 88 |
+
def sum_usage_by_model(
|
| 89 |
+
usage_by_model: dict[str, dict[str, Any]] | None,
|
| 90 |
+
) -> tuple[int, float | None]:
|
| 91 |
+
"""Sum token and recorded cost counters across per-model buckets."""
|
| 92 |
+
tokens = 0
|
| 93 |
+
cost = 0.0
|
| 94 |
+
has_cost = False
|
| 95 |
+
for usage in (usage_by_model or {}).values():
|
| 96 |
+
if not isinstance(usage, dict):
|
| 97 |
+
continue
|
| 98 |
+
tokens += _usage_tokens(usage)
|
| 99 |
+
amount = usage.get("cost_usd")
|
| 100 |
+
if isinstance(amount, (int, float)) and float(amount) > 0:
|
| 101 |
+
cost += float(amount)
|
| 102 |
+
has_cost = True
|
| 103 |
+
return tokens, (cost if has_cost else None)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def entry_recorded_totals(entry: LeaderboardEntry) -> tuple[int | None, float | None]:
|
| 107 |
+
"""Return authoritative totals from per-model buckets, falling back to run totals."""
|
| 108 |
+
tokens, cost = sum_usage_by_model(entry.token_usage_by_model)
|
| 109 |
+
total_tokens = tokens if tokens > 0 else entry.total_tokens
|
| 110 |
+
total_cost = cost if cost is not None and cost > 0 else entry.total_cost_usd
|
| 111 |
+
return total_tokens, total_cost
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def reconcile_entry_usage_totals(entry: LeaderboardEntry) -> None:
|
| 115 |
+
"""Keep run totals aligned with summed token_usage_by_model when recorded."""
|
| 116 |
+
by_model = entry.token_usage_by_model or {}
|
| 117 |
+
if not by_model:
|
| 118 |
+
return
|
| 119 |
+
tokens, cost = sum_usage_by_model(by_model)
|
| 120 |
+
if tokens > 0:
|
| 121 |
+
entry.total_tokens = tokens
|
| 122 |
+
if cost is not None and cost > 0:
|
| 123 |
+
entry.total_cost_usd = cost
|
| 124 |
+
if entry.total_tasks:
|
| 125 |
+
entry.avg_cost_per_task_usd = cost / entry.total_tasks
|
| 126 |
+
|
| 127 |
+
|
| 128 |
def _short_model_name(model: str) -> str:
|
| 129 |
return model.split("/")[-1] if "/" in model else model
|
| 130 |
|
|
|
|
| 202 |
return short
|
| 203 |
|
| 204 |
|
| 205 |
+
def _ouroboros_slot_display_lines(entry: LeaderboardEntry) -> list[str]:
|
| 206 |
index = _model_roles_index(entry.ouroboros_model_slots)
|
| 207 |
if not index:
|
| 208 |
return []
|
| 209 |
+
|
| 210 |
+
total_tokens, total_cost = entry_recorded_totals(entry)
|
| 211 |
+
total_bits: list[str] = []
|
| 212 |
+
if total_tokens:
|
| 213 |
+
total_bits.append(fmt_tokens(total_tokens))
|
| 214 |
+
if total_cost:
|
| 215 |
+
total_bits.append(fmt_cost(total_cost))
|
| 216 |
+
total_line = " · ".join(total_bits)
|
| 217 |
+
|
| 218 |
if len(index) == 1:
|
| 219 |
+
short = _short_model_name(next(iter(index)))
|
| 220 |
+
if total_line:
|
| 221 |
+
return [f"{short} {total_line}"]
|
| 222 |
+
return [short]
|
| 223 |
+
|
| 224 |
primary = entry.model
|
| 225 |
+
lines = [
|
| 226 |
_compact_slot_label(model, index[model], primary=primary)
|
| 227 |
for model in sorted(index, key=lambda name: (name != primary, name))
|
| 228 |
]
|
| 229 |
+
if total_line:
|
| 230 |
+
lines.append(f"{total_line} total")
|
| 231 |
+
return lines
|
| 232 |
|
| 233 |
|
| 234 |
def _format_token_usage_breakdown(entry: LeaderboardEntry) -> str:
|
|
|
|
| 265 |
|
| 266 |
|
| 267 |
def _format_ouroboros_slots_breakdown(entry: LeaderboardEntry) -> str:
|
| 268 |
+
lines = _ouroboros_slot_display_lines(entry)
|
| 269 |
if not lines:
|
| 270 |
return "—"
|
| 271 |
return " · ".join(lines)
|
|
|
|
| 286 |
text = _format_token_usage_breakdown(entry)
|
| 287 |
return f'<span class="wab-token-split" title="{html.escape(text)}">{html.escape(text)}</span>'
|
| 288 |
|
| 289 |
+
lines = _ouroboros_slot_display_lines(entry)
|
| 290 |
if not lines:
|
| 291 |
return "—"
|
| 292 |
title = " · ".join(lines)
|
| 293 |
+
body_parts: list[str] = []
|
| 294 |
+
for line in lines:
|
| 295 |
+
css_class = "wab-token-split-total" if line.endswith(" total") else "wab-token-split-line"
|
| 296 |
+
body_parts.append(f'<span class="{css_class}">{html.escape(line)}</span>')
|
| 297 |
+
body = "".join(body_parts)
|
| 298 |
return f'<span class="wab-token-split" title="{html.escape(title)}">{body}</span>'
|
| 299 |
|
| 300 |
|
|
|
|
| 324 |
entry = LeaderboardEntry(
|
| 325 |
model=primary_model,
|
| 326 |
harness=harness,
|
| 327 |
+
token_usage_by_model=compact_token_usage_by_model(by_model if isinstance(by_model, dict) else None),
|
| 328 |
ouroboros_model_slots=slots,
|
| 329 |
)
|
| 330 |
if _has_ouroboros_slot_split(entry):
|