Spaces:
Running
Running
Download src/traces.py from ai-forever/WebPageBench: direct link, hf CLI and curl.
- Browser
- Download file 39.6 kB
-
https://huggingface.co/spaces/ai-forever/WebPageBench/resolve/main/src/traces.py
- Command line
-
hf download hf://spaces/ai-forever/WebPageBench/src/traces.py
-
curl -L -o traces.py https://huggingface.co/spaces/ai-forever/WebPageBench/resolve/main/src/traces.py
39.6 kB
| """Agent trajectory list and detail views for the leaderboard.""" | |
| from __future__ import annotations | |
| import html | |
| import json | |
| import re | |
| from typing import Any | |
| from src.bench_config import load_bench_config | |
| from src.i18n import section_label_t, t | |
| from src.load_entries import entry_by_id, entry_label, load_raw_results | |
| from src.models import LeaderboardEntry | |
| from src.token_display import format_task_token_usage | |
| from src.ui import fmt_cost | |
| def parse_trace_nav(payload: str) -> tuple[str, str, str]: | |
| """Parse traces:<view>:<entry_id>[:<test_name>] payloads.""" | |
| parts = (payload or "").split(":", 3) | |
| if len(parts) < 2 or parts[0] != "traces": | |
| return "list", "", "" | |
| view = parts[1] if len(parts) > 1 else "list" | |
| entry_id = parts[2] if len(parts) > 2 else "" | |
| test_name = parts[3] if len(parts) > 3 else "" | |
| return view, entry_id, test_name | |
| def _esc(value: object) -> str: | |
| return html.escape(str(value if value is not None else "—")) | |
| def _truncate(text: str, limit: int = 600) -> str: | |
| text = (text or "").strip() | |
| if len(text) <= limit: | |
| return text | |
| return text[: limit - 1].rstrip() + "…" | |
| def _task_section_id(test_name: str) -> str | None: | |
| config = load_bench_config() | |
| for section_id, meta in config.get("sections", {}).items(): | |
| if test_name in (meta.get("tasks") or []): | |
| return section_id | |
| return None | |
| def _task_section_label(test_name: str) -> str: | |
| section_id = _task_section_id(test_name) | |
| if section_id: | |
| return section_label_t(section_id) | |
| return "—" | |
| def format_duration(seconds: float | int | None) -> str: | |
| if seconds is None: | |
| return "—" | |
| try: | |
| total = float(seconds) | |
| except (TypeError, ValueError): | |
| return "—" | |
| if total < 60: | |
| return f"{total:.2f}s" | |
| minutes = int(total // 60) | |
| secs = int(total % 60) | |
| if minutes < 60: | |
| return f"{minutes}m {secs}s" | |
| hours = minutes // 60 | |
| minutes = minutes % 60 | |
| return f"{hours}h {minutes}m" | |
| def _extract_url_from_state(state: object) -> str | None: | |
| if not isinstance(state, str): | |
| return None | |
| match = re.search(r"url='([^']+)'", state) | |
| return match.group(1) if match else None | |
| def _format_named_actions(actions: list[Any]) -> str: | |
| parts: list[str] = [] | |
| for item in actions: | |
| if not isinstance(item, dict): | |
| continue | |
| for name, payload in item.items(): | |
| if isinstance(payload, dict): | |
| brief = ", ".join(f"{k}={v}" for k, v in list(payload.items())[:4]) | |
| parts.append(f"{name}({brief})") | |
| else: | |
| parts.append(f"{name}({payload})") | |
| return "; ".join(parts) | |
| def _format_gui_actions(actions: list[Any]) -> str: | |
| parts: list[str] = [] | |
| for item in actions: | |
| if not isinstance(item, dict): | |
| continue | |
| desc = item.get("description") or item.get("type") | |
| if desc: | |
| parts.append(str(desc)) | |
| return "; ".join(parts) | |
| def _format_browser_action(model_output: dict[str, Any] | None) -> str: | |
| if not model_output: | |
| return "" | |
| actions = model_output.get("action") | |
| if isinstance(actions, list) and actions: | |
| formatted = _format_named_actions(actions) | |
| if formatted: | |
| return formatted | |
| gui = model_output.get("actions") | |
| if isinstance(gui, list) and gui: | |
| formatted = _format_gui_actions(gui) | |
| if formatted: | |
| return formatted | |
| description = model_output.get("action_description") | |
| if isinstance(description, str) and description.strip(): | |
| return description.strip() | |
| return "" | |
| def _normalize_agent_step(step: dict[str, Any], harness: str) -> dict[str, str]: | |
| harness = (harness or "").lower() | |
| title = t("step_prefix") | |
| body_parts: list[str] = [] | |
| action = "" | |
| url = "" | |
| if "model_output" in step: | |
| model_output = step.get("model_output") or {} | |
| title = f"{t('step_prefix')} {step.get('step', step.get('metadata', {}).get('step_number', '?'))}" | |
| action = _format_browser_action(model_output) | |
| thinking = model_output.get("thinking") or model_output.get("next_goal") | |
| if thinking: | |
| body_parts.append(str(thinking)) | |
| results = step.get("result") | |
| if isinstance(results, list): | |
| for row in results[:2]: | |
| if isinstance(row, dict) and row.get("extracted_content"): | |
| body_parts.append(str(row["extracted_content"])) | |
| url = _extract_url_from_state(step.get("state")) or "" | |
| if not body_parts and step.get("state_message"): | |
| body_parts.append(_truncate(str(step["state_message"]), 400)) | |
| elif step.get("kind") == "message": | |
| title = t("user_message") if step.get("source") == "user" else t("agent_message") | |
| body_parts.append(str(step.get("content") or "")) | |
| elif step.get("kind") == "action": | |
| title = f"{t('action')} {step.get('index', '?')}" | |
| tool = step.get("tool") or step.get("action", {}).get("kind") | |
| action = str(tool or "action") | |
| payload = step.get("action") | |
| if isinstance(payload, dict): | |
| body_parts.append(_truncate(json.dumps(payload, ensure_ascii=False), 500)) | |
| else: | |
| title = f"{t('step_prefix')} {step.get('index', step.get('step', '?'))}" | |
| body_parts.append(_truncate(json.dumps(step, ensure_ascii=False), 500)) | |
| return { | |
| "title": title, | |
| "action": action, | |
| "body": _truncate("\n\n".join(part for part in body_parts if part), 1200), | |
| "url": url, | |
| } | |
| def _step_sidebar_label(step: dict[str, Any], harness: str) -> str: | |
| if "model_output" in step: | |
| formatted = _format_browser_action(step.get("model_output") or {}) | |
| if formatted: | |
| return formatted | |
| return f"{t('step_prefix')} {step.get('step', '?')}" | |
| if step.get("kind") == "message": | |
| return t("user_message") if step.get("source") == "user" else t("agent_message") | |
| if step.get("kind") == "action": | |
| return str(step.get("tool") or "action") | |
| return f"{t('step_prefix')} {step.get('index', step.get('step', '?'))}" | |
| def _step_kind_class(step: dict[str, Any]) -> str: | |
| if step.get("kind") == "user": | |
| return "user" | |
| if step.get("kind") == "message" and step.get("source") == "user": | |
| return "user" | |
| if step.get("kind") == "action" or "model_output" in step: | |
| return "tool" | |
| return "agent" | |
| def _ui_agent_steps(test: dict[str, Any], harness: str) -> list[dict[str, str]]: | |
| trajectory = test.get("trajectory") or {} | |
| agent_steps = (trajectory.get("agent") or {}).get("steps") or [] | |
| rows: list[dict[str, str]] = [ | |
| { | |
| "id": "0", | |
| "label": t("user_message"), | |
| "kind": "user", | |
| "title": t("task_title"), | |
| "action": "", | |
| "body": str(test.get("input") or ""), | |
| "url": str(test.get("task_url") or ""), | |
| } | |
| ] | |
| for raw_step in agent_steps: | |
| if not isinstance(raw_step, dict): | |
| continue | |
| if raw_step.get("kind") == "message" and raw_step.get("source") == "user": | |
| continue | |
| norm = _normalize_agent_step(raw_step, harness) | |
| rows.append( | |
| { | |
| "id": str(len(rows)), | |
| "label": _step_sidebar_label(raw_step, harness), | |
| "kind": _step_kind_class(raw_step), | |
| "title": norm["title"], | |
| "action": norm["action"], | |
| "body": norm["body"], | |
| "url": norm["url"], | |
| } | |
| ) | |
| return rows | |
| def _extract_action_rows(test: dict[str, Any], harness: str) -> list[dict[str, str]]: | |
| rows: list[dict[str, str]] = [] | |
| for step in _ui_agent_steps(test, harness)[1:]: | |
| if step["action"]: | |
| rows.append({"step": step["id"], "name": step["label"], "detail": step["action"]}) | |
| extended = test.get("extended_metrics") or {} | |
| if not rows and extended.get("action_count"): | |
| rows.append( | |
| { | |
| "step": "—", | |
| "name": "summary", | |
| "detail": f"action_count={extended.get('action_count')}, redundancy={extended.get('action_redundancy')}", | |
| } | |
| ) | |
| return rows | |
| def _coerce_positive_cost(value: object) -> float | None: | |
| if isinstance(value, bool) or not isinstance(value, (int, float)): | |
| return None | |
| amount = float(value) | |
| return amount if amount > 0 else None | |
| def _task_cost_usd(test: dict[str, Any], model: str) -> float | None: | |
| """Return per-task LLM cost from recorded usage or pricing table.""" | |
| usage_by_model = test.get("token_usage_by_model") | |
| if isinstance(usage_by_model, dict) and usage_by_model: | |
| total = 0.0 | |
| priced = False | |
| for usage in usage_by_model.values(): | |
| if not isinstance(usage, dict): | |
| continue | |
| cost = _coerce_positive_cost(usage.get("cost_usd")) | |
| if cost is not None: | |
| total += cost | |
| priced = True | |
| if priced: | |
| return total | |
| for container in ( | |
| test, | |
| test.get("token_usage") if isinstance(test.get("token_usage"), dict) else None, | |
| ): | |
| if not isinstance(container, dict): | |
| continue | |
| for key in ("cost_usd", "total_cost_usd", "cost", "total_cost"): | |
| cost = _coerce_positive_cost(container.get(key)) | |
| if cost is not None: | |
| return cost | |
| usage = test.get("token_usage") | |
| if not isinstance(usage, dict): | |
| return None | |
| try: | |
| from src.export_results import _ensure_repo_on_path | |
| _ensure_repo_on_path() | |
| from bench_eval.llm_cost import compute_token_cost_usd, load_pricing_table | |
| from bench_eval.token_usage import normalize_token_usage | |
| normalized = normalize_token_usage(usage) | |
| cost, _ = compute_token_cost_usd( | |
| model=model, | |
| prompt_tokens=normalized["prompt_tokens"], | |
| completion_tokens=normalized["completion_tokens"], | |
| total_tokens=normalized["total_tokens"], | |
| pricing_table=load_pricing_table(), | |
| ) | |
| return cost if cost is not None and cost > 0 else None | |
| except Exception: | |
| return None | |
| def _task_token_total(test: dict[str, Any]) -> float | None: | |
| usage = test.get("token_usage") | |
| if not isinstance(usage, dict): | |
| agent = test.get("agent") | |
| usage = agent.get("token_usage") if isinstance(agent, dict) else None | |
| if not isinstance(usage, dict): | |
| return None | |
| total = usage.get("total_tokens") | |
| if isinstance(total, (int, float)) and not isinstance(total, bool) and float(total) > 0: | |
| return float(total) | |
| prompt = usage.get("prompt_tokens") | |
| completion = usage.get("completion_tokens") | |
| prompt_n = float(prompt) if isinstance(prompt, (int, float)) and not isinstance(prompt, bool) else 0.0 | |
| completion_n = float(completion) if isinstance(completion, (int, float)) and not isinstance(completion, bool) else 0.0 | |
| summed = prompt_n + completion_n | |
| return summed if summed > 0 else None | |
| def _cost_share_from_run(test: dict[str, Any], entry: LeaderboardEntry) -> float | None: | |
| """Attribute the run total by this task's share of tokens. | |
| Most submissions store only the run total. Per-task token counts still | |
| add up to that total, so the task cost is total_cost * task_tokens / run_tokens. | |
| """ | |
| total_cost = entry.total_cost_usd | |
| total_tokens = entry.total_tokens | |
| if total_cost is None or total_cost <= 0 or not total_tokens: | |
| return None | |
| task_tokens = _task_token_total(test) | |
| if not task_tokens: | |
| return None | |
| return float(total_cost) * task_tokens / float(total_tokens) | |
| def _format_tokens(token_usage: dict[str, Any] | None) -> str: | |
| if not token_usage: | |
| return "—" | |
| prompt = token_usage.get("prompt_tokens") | |
| completion = token_usage.get("completion_tokens") | |
| if prompt is None and completion is None: | |
| total = token_usage.get("total_tokens") | |
| return str(total) if total is not None else "—" | |
| prompt_s = f"{float(prompt) / 1000:.2f}K" if isinstance(prompt, (int, float)) and prompt >= 1000 else str(prompt or 0) | |
| completion_s = ( | |
| f"{float(completion) / 1000:.2f}K" if isinstance(completion, (int, float)) and completion >= 1000 else str(completion or 0) | |
| ) | |
| return t("tokens_io", prompt=prompt_s, completion=completion_s) | |
| def _meta_cell(icon: str, label: str, value: object) -> str: | |
| return ( | |
| '<div class="wab-trace-meta-cell">' | |
| f'<div class="wab-trace-meta-label"><span class="wab-trace-meta-icon">{icon}</span>{_esc(label)}</div>' | |
| f'<div class="wab-trace-meta-value">{_esc(value)}</div>' | |
| "</div>" | |
| ) | |
| def _trace_slug(entry: LeaderboardEntry, test_name: str) -> str: | |
| short_model = entry.model.split("/")[-1] if "/" in entry.model else entry.model | |
| return f"{test_name}-{entry.harness}-{short_model}" | |
| def _detail_tab_button(tab_id: str, label: str, count: int | None = None, *, active: bool = False) -> str: | |
| count_html = f'<span class="wab-trace-tab-count">{count}</span>' if count is not None else "" | |
| active_cls = " active" if active else "" | |
| return ( | |
| f'<button type="button" class="wab-trace-detail-tab{active_cls}" ' | |
| f'data-wab-trace-tab="{html.escape(tab_id)}">{html.escape(label)}{count_html}</button>' | |
| ) | |
| def trace_summaries_for_entry(entry: LeaderboardEntry) -> list[dict[str, Any]]: | |
| raw = load_raw_results(entry) | |
| if not raw: | |
| return [] | |
| rows: list[dict[str, Any]] = [] | |
| for test in raw.get("tests") or []: | |
| if not isinstance(test, dict): | |
| continue | |
| test_name = str(test.get("test_name") or "") | |
| if not test_name: | |
| continue | |
| trajectory = test.get("trajectory") or {} | |
| agent_steps = (trajectory.get("agent") or {}).get("steps") or [] | |
| dab_events = trajectory.get("dab_events") or [] | |
| ui = test.get("ui_taxonomy") or {} | |
| rows.append( | |
| { | |
| "test_name": test_name, | |
| "section": _task_section_label(test_name), | |
| "success": bool(test.get("success")), | |
| "duration": test.get("duration_seconds"), | |
| "agent_steps": test.get("agent_steps") or len(agent_steps), | |
| "dab_events": len(dab_events), | |
| "input": str(test.get("input") or ""), | |
| "domain": ui.get("domain") or "", | |
| "primary_class": (ui.get("checked_classes") or [None])[0], | |
| } | |
| ) | |
| rows.sort(key=lambda row: row["test_name"]) | |
| return rows | |
| def _status_badge(success: bool) -> str: | |
| css = "wab-trace-status pass" if success else "wab-trace-status fail" | |
| label = t("pass") if success else t("fail") | |
| return f'<span class="{css}">{label}</span>' | |
| def _trace_pill(entry_id: str, inner: str, *, active: bool = False) -> str: | |
| active_cls = " active" if active else "" | |
| return ( | |
| f'<button type="button" class="wab-trace-pill{active_cls}" ' | |
| f'data-wab-trace-filter="{html.escape(entry_id)}">{inner}</button>' | |
| ) | |
| def _trace_card(entry_id: str, test_name: str, inner: str, *, visible: bool = True) -> str: | |
| hidden = "" if visible else ' style="display:none"' | |
| payload = f"traces:detail:{entry_id}:{test_name}" | |
| return ( | |
| f'<button type="button" class="wab-trace-card" data-wab-entry-id="{html.escape(entry_id)}"' | |
| f' data-wab-trace="{html.escape(payload)}"{hidden}>' | |
| f"{inner}</button>" | |
| ) | |
| def _trace_link(payload: str, inner: str, css_class: str = "wab-trace-link") -> str: | |
| return ( | |
| f'<button type="button" class="{css_class}" data-wab-trace="{html.escape(payload)}">' | |
| f"{inner}</button>" | |
| ) | |
| def build_traces_shell( | |
| entries: list[LeaderboardEntry], | |
| *, | |
| view: str = "list", | |
| entry_id: str = "", | |
| test_name: str = "", | |
| ) -> str: | |
| if not entries: | |
| return ( | |
| '<div class="wab-traces">' | |
| '<h2 class="wab-traces-title">Agent Traces</h2>' | |
| '<p class="wab-detail-note">No submissions yet. Export results after eval.</p>' | |
| "</div>" | |
| ) | |
| if view == "detail" and entry_id and test_name: | |
| return build_trace_detail(entries, entry_id, test_name) | |
| selected = entry_by_id(entries, entry_id) or entries[0] | |
| selected_id = selected.entry_id | |
| return _build_traces_list(entries, selected_id) | |
| def _build_traces_list(entries: list[LeaderboardEntry], selected_id: str) -> str: | |
| selected_entry = entry_by_id(entries, selected_id) or entries[0] | |
| selected_summaries = trace_summaries_for_entry(selected_entry) | |
| pills: list[str] = [] | |
| for entry in entries: | |
| count = ( | |
| len(selected_summaries) | |
| if entry.entry_id == selected_entry.entry_id | |
| else int(entry.total_tasks or 0) | |
| ) | |
| active = entry.entry_id == selected_id | |
| label = html.escape(entry.model.split("/")[-1] if "/" in entry.model else entry.model) | |
| harness = html.escape(entry.harness) | |
| inner = ( | |
| f'<span class="wab-trace-pill-model">{label}</span>' | |
| f'<span class="wab-trace-pill-harness">{harness}</span>' | |
| f'<span class="wab-trace-pill-count">{count}</span>' | |
| ) | |
| pills.append(_trace_pill(entry.entry_id, inner, active=active)) | |
| cards: list[str] = [] | |
| for row in selected_summaries: | |
| test_name = row["test_name"] | |
| short_name = test_name.replace("bench_", "", 1) if test_name.startswith("bench_") else test_name | |
| tags = [] | |
| if row["domain"]: | |
| tags.append(f'<span class="wab-badge domain">{_esc(row["domain"])}</span>') | |
| if row["primary_class"]: | |
| tags.append(f'<span class="wab-badge taxonomy">{_esc(row["primary_class"])}</span>') | |
| tags_html = "".join(tags) | |
| card_inner = ( | |
| f'<div class="wab-trace-card-accent {"pass" if row["success"] else "fail"}"></div>' | |
| f'<div class="wab-trace-card-body">' | |
| f'<div class="wab-trace-card-top">' | |
| f'<span class="wab-trace-card-id">#{_esc(short_name)}</span>' | |
| f'<span class="wab-trace-card-section">{_esc(row["section"])}</span>' | |
| f"{_status_badge(row['success'])}" | |
| f'<span class="wab-trace-card-duration">⏱ {_esc(format_duration(row["duration"]))}</span>' | |
| f'<span class="wab-trace-card-steps">{row["agent_steps"]} steps · {row["dab_events"]} checks</span>' | |
| f"</div>" | |
| f'<p class="wab-trace-card-task">{_esc(_truncate(row["input"], 220))}</p>' | |
| f'<div class="wab-trace-card-tags">{tags_html}</div>' | |
| f"</div>" | |
| ) | |
| cards.append(_trace_card(selected_entry.entry_id, test_name, card_inner)) | |
| body = "".join(cards) if cards else '<p class="wab-detail-note">No task traces in this run.</p>' | |
| return ( | |
| '<div class="wab-traces">' | |
| '<h2 class="wab-traces-title">Agent Traces</h2>' | |
| '<p class="wab-traces-subtitle">Step-by-step agent actions and condition checks from every eval run.</p>' | |
| f'<div class="wab-trace-pills">{"".join(pills)}</div>' | |
| f'<div class="wab-trace-cards">{body}</div>' | |
| "</div>" | |
| ) | |
| def _find_test(entry: LeaderboardEntry, test_name: str) -> dict[str, Any] | None: | |
| raw = load_raw_results(entry) | |
| if not raw: | |
| return None | |
| for test in raw.get("tests") or []: | |
| if isinstance(test, dict) and test.get("test_name") == test_name: | |
| return test | |
| return None | |
| def build_trace_detail( | |
| entries: list[LeaderboardEntry], | |
| entry_id: str, | |
| test_name: str, | |
| *, | |
| embedded: bool = False, | |
| ) -> str: | |
| entry = entry_by_id(entries, entry_id) | |
| if entry is None: | |
| return f'<div class="wab-traces"><p class="wab-detail-note">{_esc(t("run_not_found"))}</p></div>' | |
| test = _find_test(entry, test_name) | |
| if test is None: | |
| return f'<div class="wab-traces"><p class="wab-detail-note">{_esc(t("trace_not_found"))}</p></div>' | |
| raw = load_raw_results(entry) or {} | |
| run = raw.get("run") or {} | |
| trajectory = test.get("trajectory") or {} | |
| dab_events = trajectory.get("dab_events") or [] | |
| harness = entry.harness or (test.get("agent_harness") or "") | |
| ui = test.get("ui_taxonomy") or {} | |
| dab_check = test.get("dab_check") or {} | |
| agent = test.get("agent") or {} | |
| ui_steps = _ui_agent_steps(test, harness) | |
| action_rows = _extract_action_rows(test, harness) | |
| short_name = test_name.replace("bench_", "", 1) if test_name.startswith("bench_") else test_name | |
| slug = _trace_slug(entry, test_name) | |
| provider = entry.provider or run.get("provider") or "—" | |
| domain = ui.get("domain") or "—" | |
| section = _task_section_label(test_name) | |
| token_usage = test.get("token_usage") or agent.get("token_usage") or {} | |
| task_cost = _task_cost_usd(test, entry.model) | |
| if task_cost is None: | |
| task_cost = _cost_share_from_run(test, entry) | |
| model_usage = format_task_token_usage( | |
| test, | |
| primary_model=entry.model, | |
| ouroboros_model_slots=entry.ouroboros_model_slots, | |
| ) | |
| success = bool(test.get("success")) | |
| meta_grid = "".join( | |
| [ | |
| _meta_cell("#", t("task"), f"#{short_name}"), | |
| _meta_cell("🧪", t("test_case"), test_name), | |
| _meta_cell("🤖", t("model"), entry.model), | |
| _meta_cell("☁", t("provider"), provider), | |
| _meta_cell("🏷", t("category"), f"{section} / {domain}"), | |
| _meta_cell("🌐", t("harness"), harness), | |
| _meta_cell("🖼", t("input"), entry.input_modality), | |
| _meta_cell("👣", t("steps"), test.get("agent_steps") or max(0, len(ui_steps) - 1)), | |
| _meta_cell("🪙", t("tokens"), _format_tokens(token_usage)), | |
| _meta_cell("🧩", t("model_usage"), model_usage), | |
| _meta_cell("💰", t("cost"), fmt_cost(task_cost)), | |
| _meta_cell("⏱", t("duration"), format_duration(test.get("duration_seconds"))), | |
| _meta_cell("🚩", t("result"), t("pass") if success else t("fail")), | |
| ] | |
| ) | |
| step_nav_items: list[str] = [] | |
| step_panels: list[str] = [] | |
| for step in ui_steps: | |
| active = " active" if step["id"] == "0" else "" | |
| step_nav_items.append( | |
| f'<button type="button" class="wab-trace-step-item{active}" data-wab-trace-step="{html.escape(step["id"])}" ' | |
| f'data-wab-step-label="{html.escape(step["label"].lower())}">' | |
| f'<span class="wab-trace-step-num">{int(step["id"]) + 1}</span>' | |
| f'<span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>' | |
| f'<span class="wab-trace-step-label">{_esc(step["label"])}</span>' | |
| "</button>" | |
| ) | |
| url_line = f'<div class="wab-trace-step-url">{_esc(step["url"])}</div>' if step["url"] else "" | |
| action_line = ( | |
| f'<div class="wab-trace-step-action-line"><strong>{_esc(t("action"))}:</strong> {_esc(step["action"])}</div>' | |
| if step["action"] | |
| else "" | |
| ) | |
| step_panels.append( | |
| f'<div class="wab-trace-step-panel{active}" data-wab-trace-step-panel="{html.escape(step["id"])}">' | |
| f'<div class="wab-trace-step-panel-head"><span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>' | |
| f"<strong>{_esc(step['title'])}</strong></div>" | |
| f"{url_line}{action_line}" | |
| f'<pre class="wab-trace-step-body">{_esc(step["body"])}</pre>' | |
| "</div>" | |
| ) | |
| action_cards = [] | |
| for row in action_rows: | |
| action_cards.append( | |
| '<article class="wab-trace-dab-event">' | |
| f'<div class="wab-trace-dab-head"><strong>{_esc(t("step_prefix"))} {row["step"]}: {_esc(row["name"])}</strong></div>' | |
| f'<pre class="wab-trace-step-body">{_esc(row["detail"])}</pre>' | |
| "</article>" | |
| ) | |
| dab_cards = [] | |
| for event in dab_events[:120]: | |
| if not isinstance(event, dict): | |
| continue | |
| event_name = event.get("event_name") or event.get("event_data", {}).get("type") or "event" | |
| event_data = event.get("event_data") or {} | |
| summary = _truncate(json.dumps(event_data, ensure_ascii=False), 800) | |
| dab_cards.append( | |
| '<article class="wab-trace-dab-event">' | |
| f'<div class="wab-trace-dab-head"><strong>{_esc(event_name)}</strong>' | |
| f'<span>{_esc(event.get("event_ts"))}</span></div>' | |
| f'<pre class="wab-trace-step-body">{_esc(summary)}</pre>' | |
| "</article>" | |
| ) | |
| checks = dab_check.get("check_results") or [] | |
| check_rows = [] | |
| for row in checks: | |
| if not isinstance(row, dict): | |
| continue | |
| ok = _condition_ok(row) | |
| mark = "✓" if ok else "✗" | |
| css = "wab-trace-check-pass" if ok else "wab-trace-check-fail" | |
| row_cls = "pass" if ok else "fail" | |
| extra = f" [group:{row.get('group')}]" if row.get("group") else "" | |
| check_rows.append( | |
| f'<li class="{row_cls}"><span class="{css}">{mark}</span> ' | |
| f"{_esc(_condition_label(row))}{_esc(extra)}</li>" | |
| ) | |
| checks_html = f'<ul class="tv-check-list">{"".join(check_rows)}</ul>' if check_rows else "<p>—</p>" | |
| export_json = json.dumps(test, ensure_ascii=False).replace("</", "<\\/") | |
| tabs = "".join( | |
| [ | |
| _detail_tab_button("agent", t("agent_tab"), len(ui_steps), active=True), | |
| _detail_tab_button("actions", t("actions_tab"), len(action_rows) or None), | |
| _detail_tab_button("requests", t("requests_tab"), len(dab_events) or None), | |
| _detail_tab_button("eval", t("eval_tab"), len(checks) or None), | |
| _detail_tab_button("logs", t("logs_tab")), | |
| ] | |
| ) | |
| actions_body = "".join(action_cards) if action_cards else f'<p class="wab-detail-note">{_esc(t("no_actions"))}</p>' | |
| requests_body = "".join(dab_cards) if dab_cards else f'<p class="wab-detail-note">{_esc(t("no_checks"))}</p>' | |
| if embedded: | |
| crumb = f'<nav class="wab-trace-crumb"><span class="wab-trace-crumb-current">{_esc(slug)}</span></nav>' | |
| else: | |
| back = _trace_link(f"traces:list:{entry_id}", "← Traces", "wab-trace-back") | |
| crumb = ( | |
| f'<nav class="wab-trace-crumb">{back}<span class="wab-trace-crumb-sep">›</span>' | |
| f'<span class="wab-trace-crumb-current">{_esc(slug)}</span></nav>' | |
| ) | |
| return ( | |
| f'<div class="wab-traces wab-trace-detail-view" id="wab-trace-detail">' | |
| f"{crumb}" | |
| '<header class="wab-trace-hero-card">' | |
| f'<p class="wab-trace-hero-task">{_esc(test.get("input") or "")}</p>' | |
| f'<div class="wab-trace-meta-grid">{meta_grid}</div>' | |
| '<div class="wab-trace-hero-foot">' | |
| f"{_status_badge(success)}" | |
| f'<button type="button" class="wab-trace-export" data-wab-trace-export="1">⬇ {_esc(t("export_json"))}</button>' | |
| "</div>" | |
| f'<script type="application/json" class="wab-trace-export-data">{export_json}</script>' | |
| "</header>" | |
| f'<div class="wab-trace-detail-tabs">{tabs}</div>' | |
| '<div class="wab-trace-detail-panels">' | |
| '<section class="wab-trace-detail-panel active" data-wab-trace-panel="agent">' | |
| '<div class="wab-trace-split">' | |
| '<aside class="wab-trace-step-sidebar">' | |
| '<div class="wab-trace-step-sidebar-head">' | |
| f'<input type="text" class="wab-trace-step-filter" placeholder="{_esc(t("filter_steps"))}" />' | |
| f'<span class="wab-trace-step-total">{_esc(t("steps_count", n=len(ui_steps)))}</span>' | |
| "</div>" | |
| f'<div class="wab-trace-step-nav">{"".join(step_nav_items)}</div>' | |
| "</aside>" | |
| f'<main class="wab-trace-step-main">{"".join(step_panels)}</main>' | |
| "</div>" | |
| "</section>" | |
| '<section class="wab-trace-detail-panel" data-wab-trace-panel="actions">' | |
| f'<div class="wab-trace-dab-list">{actions_body}</div>' | |
| "</section>" | |
| '<section class="wab-trace-detail-panel" data-wab-trace-panel="requests">' | |
| f'<div class="wab-trace-dab-list">{requests_body}</div>' | |
| "</section>" | |
| '<section class="wab-trace-detail-panel" data-wab-trace-panel="eval">' | |
| f"<p><strong>all_passed:</strong> {_esc(dab_check.get('all_passed'))}</p>" | |
| f"{checks_html}" | |
| f"<p><strong>failed_conditions:</strong> {_esc(', '.join(dab_check.get('failed_conditions') or []) or '—')}</p>" | |
| "</section>" | |
| '<section class="wab-trace-detail-panel" data-wab-trace-panel="logs">' | |
| f"<h4>{_esc(t('agent_output'))}</h4>" | |
| f'<pre class="wab-trace-output">{_esc(_truncate(str(test.get("actual_output") or ""), 6000))}</pre>' | |
| f"<h4>{_esc(t('final_result'))}</h4>" | |
| f'<pre class="wab-trace-output">{_esc(_truncate(str(agent.get("final_result") or ""), 4000))}</pre>' | |
| f"<h4>{_esc(t('agent_error'))}</h4>" | |
| f'<pre class="wab-trace-output">{_esc(test.get("agent_error") or "—")}</pre>' | |
| "</section>" | |
| "</div>" | |
| "</div>" | |
| ) | |
| def _condition_ok(row: dict) -> bool: | |
| if row.get("group"): | |
| value = row.get("group_success") | |
| if value is None: | |
| value = row.get("passed") | |
| return bool(value) | |
| if "success" in row: | |
| return bool(row.get("success")) | |
| return bool(row.get("passed")) | |
| def _condition_label(row: dict) -> str: | |
| return str(row.get("description") or row.get("event_name") or row.get("condition") or row.get("name") or "condition") | |
| def _meta_plain(label: str, value: object, *, html_value: str | None = None) -> str: | |
| shown = html_value if html_value is not None else _esc(value) | |
| return ( | |
| '<div class="wab-trace-meta-cell">' | |
| f'<div class="wab-trace-meta-label">{_esc(label)}</div>' | |
| f'<div class="wab-trace-meta-value">{shown}</div>' | |
| "</div>" | |
| ) | |
| def _run_header(entry: LeaderboardEntry) -> str: | |
| success = f"{entry.success_pct}% ({entry.passed_tasks}/{entry.total_tasks})" | |
| return ( | |
| '<header class="wab-trace-hero-card wab-run-header">' | |
| f'<p class="wab-trace-hero-task">{_esc(entry.model)} · {_esc(entry.harness)}</p>' | |
| '<div class="wab-trace-meta-grid">' | |
| f"{_meta_plain(t('success'), success)}" | |
| f"{_meta_plain(t('input'), entry.input_modality)}" | |
| f"{_meta_plain(t('duration'), format_duration(entry.total_duration_seconds))}" | |
| f"{_meta_plain(t('cost'), fmt_cost(entry.total_cost_usd))}" | |
| "</div>" | |
| "</header>" | |
| ) | |
| def _run_task_detail(entry: LeaderboardEntry, test_name: str) -> str: | |
| test = _find_test(entry, test_name) | |
| if test is None: | |
| return f'<p class="wab-detail-note">{_esc(t("task_not_found"))}</p>' | |
| trajectory = test.get("trajectory") or {} | |
| events = [event for event in (trajectory.get("dab_events") or []) if isinstance(event, dict)] | |
| names = [str(name) for name in (trajectory.get("dab_event_names") or [])] | |
| if not names: | |
| names = [str(event.get("event_name") or "event") for event in events] | |
| check = test.get("dab_check") or {} | |
| results = [row for row in (check.get("check_results") or []) if isinstance(row, dict)] | |
| if results: | |
| items = [] | |
| for row in results: | |
| ok = _condition_ok(row) | |
| mark = "✓" if ok else "✗" | |
| css = "wab-trace-check-pass" if ok else "wab-trace-check-fail" | |
| extra = f" [group:{row.get('group')}]" if row.get("group") else "" | |
| row_cls = "pass" if ok else "fail" | |
| items.append( | |
| f'<li class="{row_cls}"><span class="{css}">{mark}</span> {_esc(_condition_label(row))}{_esc(extra)}</li>' | |
| ) | |
| checks_html = f'<ul class="tv-check-list">{"".join(items)}</ul>' | |
| else: | |
| checks_html = "<p>—</p>" | |
| shown = events[:80] | |
| cards = [] | |
| for event in shown: | |
| name = event.get("event_name") or "event" | |
| payload = _truncate(json.dumps(event.get("event_data") or {}, ensure_ascii=False, indent=2), 2000) | |
| cards.append( | |
| '<article class="wab-trace-dab-event">' | |
| '<div class="wab-trace-dab-head">' | |
| f"<strong>{_esc(name)}</strong>" | |
| f"<span>{_esc(event.get('event_ts') or '')}</span>" | |
| "</div>" | |
| f'<pre class="wab-trace-step-body">{_esc(payload)}</pre>' | |
| "</article>" | |
| ) | |
| extra_note = "" | |
| if len(events) > len(shown): | |
| extra_note = f'<p class="wab-detail-note">{_esc(t("showing_events", shown=len(shown), total=len(events)))}</p>' | |
| event_html = "".join(cards) if cards else f'<p class="wab-detail-note">{_esc(t("no_events"))}</p>' | |
| prompt = test.get("task") or test.get("input") or "" | |
| if check.get("all_passed") is True: | |
| result = "PASS" | |
| elif check: | |
| result = "FAIL" | |
| else: | |
| result = "PASS" if test.get("success") else "FAIL" | |
| passed = check.get("passed") | |
| total = check.get("total") | |
| result_badge = _status_badge(result == "PASS") | |
| result_html = result_badge if passed is None else f"{result_badge} {_esc(f'{passed}/{total}')}" | |
| return ( | |
| '<header class="wab-trace-hero-card">' | |
| f'<p class="wab-trace-hero-task">{_esc(prompt)}</p>' | |
| '<div class="wab-trace-meta-grid">' | |
| f"{_meta_plain(t('test_case'), test_name)}" | |
| f"{_meta_plain(t('result'), '', html_value=result_html)}" | |
| f"{_meta_plain(t('events'), len(events))}" | |
| f"{_meta_plain(t('duration'), format_duration(test.get('duration_seconds')))}" | |
| "</div>" | |
| "</header>" | |
| f"<h4>{_esc(t('conditions'))}</h4>" | |
| f"{checks_html}" | |
| f"<h4>{_esc(t('event_chain'))}</h4>" | |
| f'<p class="wab-run-chain">{_esc(" → ".join(names) or "—")}</p>' | |
| f'<div class="wab-trace-dab-list">{event_html}</div>' | |
| f"{extra_note}" | |
| ) | |
| def build_run_page(entries: list[LeaderboardEntry], entry_id: str, test_name: str = "") -> str: | |
| """Run page: task list plus the selected task's prompt, conditions, and events.""" | |
| entry = entry_by_id(entries, entry_id) | |
| if entry is None: | |
| return '<p class="wab-detail-note">Run not found.</p>' | |
| summaries = trace_summaries_for_entry(entry) | |
| header = _run_header(entry) | |
| if not summaries: | |
| return ( | |
| '<div class="wab-run">' | |
| f"{header}" | |
| f'<p class="wab-detail-note">{t("task_list_unavailable")}</p>' | |
| "</div>" | |
| ) | |
| names = [row["test_name"] for row in summaries] | |
| selected = test_name if test_name in names else names[0] | |
| items = [] | |
| for row in summaries: | |
| active = " active" if row["test_name"] == selected else "" | |
| tone = " pass" if row["success"] else " fail" | |
| short = row["test_name"].replace("bench_", "", 1) if str(row["test_name"]).startswith("bench_") else row["test_name"] | |
| payload = f"submission:{entry.entry_id}:{row['test_name']}" | |
| items.append( | |
| f'<button type="button" class="tv-traj-item{tone}{active}" data-wab-nav="{html.escape(payload)}">' | |
| f"<strong>{_esc(short)}</strong>" | |
| f'<span class="meta">{_esc(row["section"])} · {row["dab_events"]} {_esc(t("events_word"))} {_status_badge(bool(row["success"]))}</span>' | |
| "</button>" | |
| ) | |
| return ( | |
| '<div class="wab-run">' | |
| f"{header}" | |
| '<div class="tv-traj-layout">' | |
| f'<aside class="tv-traj-list">{"".join(items)}</aside>' | |
| f'<div class="tv-traj-detail">{build_trace_detail(entries, entry.entry_id, selected, embedded=True)}</div>' | |
| "</div>" | |
| "</div>" | |
| ) | |
| def build_trace_js_on_load() -> str: | |
| """Gradio 6 js_on_load for trace list/detail interactions (must include 'navigate').""" | |
| return """ | |
| function filterTraceEntry(root, entryId) { | |
| if (!root || !entryId) return; | |
| root.querySelectorAll(".wab-trace-pill").forEach((pill) => { | |
| pill.classList.toggle("active", pill.getAttribute("data-wab-trace-filter") === entryId); | |
| }); | |
| root.querySelectorAll(".wab-trace-card").forEach((card) => { | |
| card.style.display = card.getAttribute("data-wab-entry-id") === entryId ? "flex" : "none"; | |
| }); | |
| } | |
| element.addEventListener("click", (event) => { | |
| const tab = event.target.closest("[data-wab-trace-tab]"); | |
| if (tab && tab.closest(".wab-trace-detail-view")) { | |
| event.preventDefault(); | |
| const root = tab.closest(".wab-trace-detail-view"); | |
| const tabId = tab.getAttribute("data-wab-trace-tab"); | |
| root.querySelectorAll("[data-wab-trace-tab]").forEach((el) => { | |
| el.classList.toggle("active", el === tab); | |
| }); | |
| root.querySelectorAll("[data-wab-trace-panel]").forEach((panel) => { | |
| panel.classList.toggle("active", panel.getAttribute("data-wab-trace-panel") === tabId); | |
| }); | |
| return; | |
| } | |
| const stepBtn = event.target.closest("[data-wab-trace-step]"); | |
| if (stepBtn && stepBtn.closest(".wab-trace-detail-view")) { | |
| event.preventDefault(); | |
| const root = stepBtn.closest(".wab-trace-detail-view"); | |
| const stepId = stepBtn.getAttribute("data-wab-trace-step"); | |
| root.querySelectorAll("[data-wab-trace-step]").forEach((el) => { | |
| el.classList.toggle("active", el === stepBtn); | |
| }); | |
| root.querySelectorAll("[data-wab-trace-step-panel]").forEach((panel) => { | |
| panel.classList.toggle("active", panel.getAttribute("data-wab-trace-step-panel") === stepId); | |
| }); | |
| return; | |
| } | |
| const exportBtn = event.target.closest("[data-wab-trace-export]"); | |
| if (exportBtn && exportBtn.closest(".wab-trace-detail-view")) { | |
| event.preventDefault(); | |
| const root = exportBtn.closest(".wab-trace-detail-view"); | |
| const dataEl = root.querySelector(".wab-trace-export-data"); | |
| if (!dataEl) return; | |
| const blob = new Blob([dataEl.textContent], { type: "application/json" }); | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement("a"); | |
| a.href = url; | |
| a.download = (root.querySelector(".wab-trace-crumb-current")?.textContent || "trace") + ".json"; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| return; | |
| } | |
| const pill = event.target.closest("[data-wab-trace-filter]"); | |
| if (pill) { | |
| event.preventDefault(); | |
| const entryId = pill.getAttribute("data-wab-trace-filter"); | |
| if (!entryId) return; | |
| trigger("navigate", { payload: `traces:list:${entryId}` }); | |
| return; | |
| } | |
| const navBtn = event.target.closest("[data-wab-trace]"); | |
| if (!navBtn) return; | |
| event.preventDefault(); | |
| const payload = navBtn.getAttribute("data-wab-trace"); | |
| if (!payload) return; | |
| if (payload.startsWith("traces:list:")) { | |
| trigger("navigate", { payload }); | |
| return; | |
| } | |
| if (window.wabOpenDetailTab) window.wabOpenDetailTab("traces"); | |
| const anchor = document.getElementById("wab-detail-anchor"); | |
| if (anchor) anchor.scrollIntoView({ behavior: "smooth", block: "start" }); | |
| trigger("navigate", { payload }); | |
| }); | |
| element.addEventListener("input", (event) => { | |
| const input = event.target.closest(".wab-trace-step-filter"); | |
| if (!input) return; | |
| const root = input.closest(".wab-trace-detail-view"); | |
| if (!root) return; | |
| const query = (input.value || "").trim().toLowerCase(); | |
| root.querySelectorAll("[data-wab-trace-step]").forEach((btn) => { | |
| const label = btn.getAttribute("data-wab-step-label") || ""; | |
| btn.style.display = !query || label.includes(query) ? "" : "none"; | |
| }); | |
| }); | |
| """ | |