Spaces:
Running
Running
| """Web Agent Bench leaderboard (Gradio Space source).""" | |
| from __future__ import annotations | |
| import os | |
| # HF Spaces enable SSR by default. It is not needed for this app and triggers asyncio | |
| # cleanup errors on Python 3.13 (Invalid file descriptor: -1). launch(ssr_mode=False) | |
| # is ignored on HF — the env var must be set before Gradio is imported. | |
| os.environ.setdefault("GRADIO_SSR_MODE", "false") | |
| import os | |
| from pathlib import Path | |
| import gradio as gr | |
| import pandas as pd | |
| from src.bench_config import load_bench_config, section_order, taxonomy_label | |
| from src.load_entries import ( | |
| FILTER_ALL, | |
| entry_choices, | |
| filter_entries, | |
| harness_filter_choices, | |
| input_filter_choices, | |
| load_entries, | |
| ) | |
| from src.display import dataframe_to_html | |
| from src.tables import ( | |
| category_description_html, | |
| section_leaderboard_dataframe, | |
| taxonomy_ids_in_entries, | |
| taxonomy_leaderboard_dataframe, | |
| ) | |
| from src.traces import build_run_page | |
| from src.ui import build_board, build_highlight_picks, build_nav_js, build_shell_top, load_css | |
| from src.formatting import fmt_pct_value, round2 | |
| from src.i18n import about_markdown, how_to_submit, section_label_t, set_lang, t | |
| def _category_table(entries, kind: str, item_id: str): | |
| if kind == "section": | |
| desc, df = category_description_html("section", item_id), section_leaderboard_dataframe(entries, item_id) | |
| else: | |
| desc, df = category_description_html("taxonomy", item_id), taxonomy_leaderboard_dataframe(entries, item_id) | |
| note = t("empty_class") if kind == "taxonomy" else t("no_data") | |
| return desc, dataframe_to_html(df, empty_note=note) | |
| def _metrics_html(entries) -> str: | |
| metric_rows = [ | |
| { | |
| t("model"): entry.model, | |
| t("harness"): entry.harness, | |
| t("input"): entry.input_modality, | |
| t("success"): entry.success_pct, | |
| t("done_pass"): ( | |
| f"{entry.strict_passed_tasks}/{entry.total_tasks} ({fmt_pct_value(entry.strict_success_rate)})" | |
| if entry.strict_passed_tasks is not None and entry.total_tasks > 0 | |
| else None | |
| ), | |
| t("tasks_col"): f"{entry.passed_tasks}/{entry.total_tasks}", | |
| t("avg_time"): round2(entry.avg_duration_seconds), | |
| t("total_time"): round2(entry.total_duration_seconds), | |
| t("total_steps"): entry.total_agent_steps, | |
| t("steps"): round2(entry.avg_agent_steps), | |
| t("tokens_per_task"): round2(entry.avg_tokens_per_task), | |
| t("tokens"): entry.total_tokens, | |
| t("total_cost"): round2(entry.total_cost_usd), | |
| } | |
| for entry in entries | |
| ] | |
| return dataframe_to_html(pd.DataFrame(metric_rows), wrap_headers=True, table_class="wab-metrics-table") | |
| def _parse_nav_payload(payload: str) -> tuple[str, str, str]: | |
| tab, rest = (payload.split(":", 1) + [""])[:2] | |
| if tab == "submission": | |
| entry_id, _, test_name = rest.partition(":") | |
| return tab, entry_id, test_name | |
| return tab, rest, "" | |
| def build_demo(entries=None): | |
| config = load_bench_config() | |
| entries = entries if entries is not None else load_entries() | |
| view_choices = [(t(f"view_{view['id']}"), view["id"]) for view in config["views"]] | |
| submission_choices = entry_choices(entries) | |
| default_submission = submission_choices[0][1] if submission_choices else None | |
| section_choices = [(section_label_t(section_id), section_id) for section_id in section_order()] | |
| present_taxonomy = taxonomy_ids_in_entries(entries) | |
| taxonomy_choices = [(taxonomy_label(class_id), class_id) for class_id in present_taxonomy] | |
| default_section = section_choices[0][1] if section_choices else None | |
| default_taxonomy = taxonomy_choices[0][1] if taxonomy_choices else None | |
| theme = gr.themes.Base( | |
| primary_hue=gr.themes.colors.orange, | |
| neutral_hue=gr.themes.colors.gray, | |
| ).set( | |
| body_background_fill="*neutral_950", | |
| body_background_fill_dark="*neutral_950", | |
| block_background_fill="*neutral_900", | |
| block_background_fill_dark="*neutral_900", | |
| block_border_color="*neutral_800", | |
| block_border_color_dark="*neutral_800", | |
| body_text_color="*neutral_50", | |
| body_text_color_dark="*neutral_50", | |
| ) | |
| with gr.Blocks(title="WebPageBench", elem_classes=["wab-root"]) as demo: | |
| lang_state = gr.State("en") | |
| shell = gr.HTML(build_shell_top(entries)) | |
| with gr.Tabs(elem_id="wab-page-tabs", elem_classes=["wab-page-tabs"]): | |
| with gr.Tab("Leaderboard", elem_classes=["wab-page-panel"]): | |
| highlights = gr.HTML(build_highlight_picks(entries), elem_classes=["wab-highlights-wrap"]) | |
| board_title = gr.HTML(f'<h2 class="wab-block-title">{t("leaderboard_h")}</h2>') | |
| with gr.Accordion("Filters", open=False, elem_classes=["wab-filters"], elem_id="wab-filters") as filters_acc: | |
| view = gr.Radio( | |
| choices=view_choices, | |
| value="success", | |
| show_label=False, | |
| elem_classes=["wab-view-nav-radio"], | |
| container=False, | |
| ) | |
| input_kicker = gr.HTML(f'<p class="wab-filter-kicker">{t("input")}</p>') | |
| input_filter = gr.Radio( | |
| choices=input_filter_choices(entries), | |
| value=FILTER_ALL, | |
| show_label=False, | |
| elem_classes=["wab-view-nav-radio", "wab-filter-radio"], | |
| container=False, | |
| ) | |
| harness_kicker = gr.HTML(f'<p class="wab-filter-kicker">{t("harness")}</p>') | |
| harness_filter = gr.Radio( | |
| choices=harness_filter_choices(entries), | |
| value=FILTER_ALL, | |
| show_label=False, | |
| elem_classes=["wab-view-nav-radio", "wab-filter-radio"], | |
| container=False, | |
| ) | |
| board = gr.HTML(build_board(entries, "success")) | |
| def _visible(input_id: str, harness_id: str): | |
| by_input = filter_entries(entries, input_id=input_id, harness_id=FILTER_ALL) | |
| harness_opts = harness_filter_choices(by_input) | |
| valid_ids = {value for _, value in harness_opts} | |
| selected_harness = harness_id if harness_id in valid_ids else FILTER_ALL | |
| return filter_entries(entries, input_id=input_id, harness_id=selected_harness), harness_opts, selected_harness | |
| def on_view_change(view_id: str, input_id: str, harness_id: str, lang: str): | |
| set_lang(lang) | |
| visible, _, _ = _visible(input_id, harness_id) | |
| return build_board(visible, view_id) | |
| def on_input_or_harness_change(view_id: str, input_id: str, harness_id: str, section_id: str, taxonomy_id: str, lang: str): | |
| set_lang(lang) | |
| visible, harness_opts, selected_harness = _visible(input_id, harness_id) | |
| section_html = _category_table(visible, "section", section_id) if section_id else ("", "") | |
| taxonomy_html = _category_table(visible, "taxonomy", taxonomy_id) if taxonomy_id else ("", "") | |
| return ( | |
| build_board(visible, view_id), | |
| gr.update(choices=harness_opts, value=selected_harness), | |
| section_html[0], | |
| section_html[1], | |
| taxonomy_html[0], | |
| taxonomy_html[1], | |
| _metrics_html(visible), | |
| ) | |
| gr.HTML('<div id="wab-detail-anchor" class="wab-detail-anchor"></div>') | |
| with gr.Tabs(elem_id="wab-detail-tabs", elem_classes=["wab-tabs"]): | |
| with gr.Tab("Details", elem_classes=["wab-tab-panel"]): | |
| submission = gr.Dropdown( | |
| choices=submission_choices, | |
| value=default_submission, | |
| label="Run", | |
| interactive=True, | |
| filterable=True, | |
| elem_id="wab-submission-dropdown", | |
| elem_classes=["wab-submission-dropdown"], | |
| ) | |
| initial_run = ( | |
| build_run_page(entries, default_submission) | |
| if default_submission | |
| else '<p class="wab-detail-note">No runs yet.</p>' | |
| ) | |
| detail_run = gr.HTML(value=initial_run, elem_classes=["wab-detail-html"]) | |
| def on_submission_change(entry_id: str, lang: str): | |
| set_lang(lang) | |
| return build_run_page(entries, entry_id or "") | |
| submission.select( | |
| on_submission_change, | |
| inputs=[submission, lang_state], | |
| outputs=[detail_run], | |
| ) | |
| with gr.Tab("Sections", elem_classes=["wab-tab-panel"]): | |
| section_note = gr.HTML(f'<p class="wab-detail-note">{t("section_note")}</p>') | |
| section_pick = gr.Radio( | |
| choices=section_choices, | |
| value=default_section, | |
| show_label=False, | |
| elem_id="wab-section-pick", | |
| elem_classes=["wab-category-radio"], | |
| container=False, | |
| ) | |
| section_initial = _category_table(entries, "section", default_section) if default_section else ("", "") | |
| section_desc = gr.HTML(value=section_initial[0], elem_classes=["wab-category-desc-wrap"]) | |
| section_table = gr.HTML(value=section_initial[1], elem_classes=["wab-detail-html"]) | |
| def on_section_change(section_id: str, input_id: str, harness_id: str, lang: str): | |
| set_lang(lang) | |
| visible, _, _ = _visible(input_id, harness_id) | |
| return _category_table(visible, "section", section_id) | |
| section_pick.change( | |
| on_section_change, | |
| inputs=[section_pick, input_filter, harness_filter, lang_state], | |
| outputs=[section_desc, section_table], | |
| ) | |
| with gr.Tab("UI taxonomy", elem_classes=["wab-tab-panel"]): | |
| taxonomy_note = gr.HTML(f'<p class="wab-detail-note">{t("taxonomy_note")}</p>') | |
| taxonomy_pick = gr.Radio( | |
| choices=taxonomy_choices, | |
| value=default_taxonomy, | |
| show_label=False, | |
| elem_id="wab-taxonomy-pick", | |
| elem_classes=["wab-category-radio"], | |
| container=False, | |
| ) | |
| taxonomy_initial = _category_table(entries, "taxonomy", default_taxonomy) if default_taxonomy else ("", "") | |
| taxonomy_desc = gr.HTML(value=taxonomy_initial[0], elem_classes=["wab-category-desc-wrap"]) | |
| taxonomy_table = gr.HTML(value=taxonomy_initial[1], elem_classes=["wab-detail-html"]) | |
| def on_taxonomy_change(class_id: str, input_id: str, harness_id: str, lang: str): | |
| set_lang(lang) | |
| visible, _, _ = _visible(input_id, harness_id) | |
| return _category_table(visible, "taxonomy", class_id) | |
| taxonomy_pick.change( | |
| on_taxonomy_change, | |
| inputs=[taxonomy_pick, input_filter, harness_filter, lang_state], | |
| outputs=[taxonomy_desc, taxonomy_table], | |
| ) | |
| with gr.Tab("Metrics", elem_classes=["wab-tab-panel"]): | |
| metrics_table = gr.HTML( | |
| value=_metrics_html(entries), | |
| elem_classes=["wab-detail-html", "wab-metrics-html"], | |
| ) | |
| with gr.Tab("About", elem_classes=["wab-page-panel"]): | |
| about_copy = gr.Markdown(about_markdown(), elem_classes=["wab-about-copy"]) | |
| submit_title = gr.HTML(f'<h2 class="wab-block-title">{t("how_to_submit")}</h2>') | |
| submit_copy = gr.Markdown(how_to_submit(), elem_classes=["wab-about-copy"]) | |
| nav_payload = gr.Textbox( | |
| value="", | |
| show_label=False, | |
| visible="hidden", | |
| elem_id="wab-nav-payload", | |
| elem_classes=["wab-nav-trigger-hidden"], | |
| ) | |
| nav_btn = gr.Button( | |
| "Navigate", | |
| visible="hidden", | |
| elem_id="wab-nav-go", | |
| elem_classes=["wab-nav-trigger-hidden"], | |
| ) | |
| lang_value = gr.Textbox( | |
| value="en", | |
| show_label=False, | |
| visible="hidden", | |
| elem_id="wab-lang-value", | |
| elem_classes=["wab-nav-trigger-hidden"], | |
| ) | |
| lang_btn = gr.Button( | |
| "Language", | |
| visible="hidden", | |
| elem_id="wab-lang-go", | |
| elem_classes=["wab-nav-trigger-hidden"], | |
| ) | |
| def on_badge_nav(payload: str = "", input_id: str = FILTER_ALL, harness_id: str = FILTER_ALL, lang: str = "en"): | |
| set_lang(lang) | |
| no_update = gr.update() | |
| if not payload: | |
| return (no_update,) * 8 | |
| visible, _, _ = _visible(input_id, harness_id) | |
| tab, category, test_name = _parse_nav_payload(payload) | |
| section_pick_up = section_desc_up = section_table_up = no_update | |
| tax_pick_up = tax_desc_up = tax_table_up = no_update | |
| submission_up = run_up = no_update | |
| if tab == "submission" and category: | |
| submission_up = gr.update(value=category) | |
| run_up = build_run_page(entries, category, test_name) | |
| elif tab == "sections" and category: | |
| section_pick_up = gr.update(value=category) | |
| desc, table = _category_table(visible, "section", category) | |
| section_desc_up = gr.update(value=desc) | |
| section_table_up = gr.update(value=table) | |
| elif tab == "taxonomy" and category: | |
| tax_pick_up = gr.update(value=category) | |
| desc, table = _category_table(visible, "taxonomy", category) | |
| tax_desc_up = gr.update(value=desc) | |
| tax_table_up = gr.update(value=table) | |
| return ( | |
| section_pick_up, | |
| section_desc_up, | |
| section_table_up, | |
| tax_pick_up, | |
| tax_desc_up, | |
| tax_table_up, | |
| submission_up, | |
| run_up, | |
| ) | |
| def on_lang( | |
| lang: str, | |
| view_id: str, | |
| input_id: str, | |
| harness_id: str, | |
| section_id: str, | |
| taxonomy_id: str, | |
| entry_id: str, | |
| ): | |
| set_lang(lang) | |
| visible, harness_opts, selected_harness = _visible(input_id, harness_id) | |
| section_html = _category_table(visible, "section", section_id) if section_id else ("", "") | |
| taxonomy_html = _category_table(visible, "taxonomy", taxonomy_id) if taxonomy_id else ("", "") | |
| run_html = ( | |
| build_run_page(entries, entry_id or "") | |
| if entry_id | |
| else f'<p class="wab-detail-note">{t("no_data")}</p>' | |
| ) | |
| localized_views = [(t(f"view_{view_row['id']}"), view_row["id"]) for view_row in config["views"]] | |
| localized_sections = [(section_label_t(section_id_row), section_id_row) for section_id_row in section_order()] | |
| return ( | |
| lang, | |
| build_shell_top(entries), | |
| build_highlight_picks(entries), | |
| f'<h2 class="wab-block-title">{t("leaderboard_h")}</h2>', | |
| gr.update(label=t("filters")), | |
| gr.update(choices=localized_views, value=view_id), | |
| f'<p class="wab-filter-kicker">{t("input")}</p>', | |
| gr.update(choices=input_filter_choices(entries), value=input_id), | |
| f'<p class="wab-filter-kicker">{t("harness")}</p>', | |
| gr.update(choices=harness_opts, value=selected_harness), | |
| build_board(visible, view_id), | |
| gr.update(label=t("run")), | |
| run_html, | |
| f'<p class="wab-detail-note">{t("section_note")}</p>', | |
| gr.update(choices=localized_sections, value=section_id), | |
| section_html[0], | |
| section_html[1], | |
| f'<p class="wab-detail-note">{t("taxonomy_note")}</p>', | |
| taxonomy_html[0], | |
| taxonomy_html[1], | |
| _metrics_html(visible), | |
| about_markdown(), | |
| f'<h2 class="wab-block-title">{t("how_to_submit")}</h2>', | |
| how_to_submit(), | |
| ) | |
| filter_inputs = [view, input_filter, harness_filter, section_pick, taxonomy_pick, lang_state] | |
| filter_outputs = [ | |
| board, | |
| harness_filter, | |
| section_desc, | |
| section_table, | |
| taxonomy_desc, | |
| taxonomy_table, | |
| metrics_table, | |
| ] | |
| view.change(on_view_change, inputs=[view, input_filter, harness_filter, lang_state], outputs=board) | |
| input_filter.change(on_input_or_harness_change, inputs=filter_inputs, outputs=filter_outputs) | |
| harness_filter.change(on_input_or_harness_change, inputs=filter_inputs, outputs=filter_outputs) | |
| lang_btn.click( | |
| on_lang, | |
| inputs=[lang_value, view, input_filter, harness_filter, section_pick, taxonomy_pick, submission], | |
| outputs=[ | |
| lang_state, | |
| shell, | |
| highlights, | |
| board_title, | |
| filters_acc, | |
| view, | |
| input_kicker, | |
| input_filter, | |
| harness_kicker, | |
| harness_filter, | |
| board, | |
| submission, | |
| detail_run, | |
| section_note, | |
| section_pick, | |
| section_desc, | |
| section_table, | |
| taxonomy_note, | |
| taxonomy_desc, | |
| taxonomy_table, | |
| metrics_table, | |
| about_copy, | |
| submit_title, | |
| submit_copy, | |
| ], | |
| ) | |
| nav_btn.click( | |
| on_badge_nav, | |
| inputs=[nav_payload, input_filter, harness_filter, lang_state], | |
| outputs=[ | |
| section_pick, | |
| section_desc, | |
| section_table, | |
| taxonomy_pick, | |
| taxonomy_desc, | |
| taxonomy_table, | |
| submission, | |
| detail_run, | |
| ], | |
| js=( | |
| "(payload, inputId, harnessId, lang) => {" | |
| "const p = window.__wabPendingNav || payload || '';" | |
| "const tab = (p.split(':')[0] || 'submission');" | |
| "if (window.wabOpenDetailTab) window.wabOpenDetailTab(tab);" | |
| "document.getElementById('wab-detail-anchor')" | |
| "?.scrollIntoView({behavior:'smooth', block:'start'});" | |
| "return [p, inputId, harnessId, lang];" | |
| "}" | |
| ), | |
| ) | |
| demo.load(lambda: None, None, None, js=build_nav_js()) | |
| return demo, theme, load_css() | |
| demo, theme, css = build_demo() | |
| if __name__ == "__main__": | |
| demo.launch(theme=theme, css=css, share=False, ssr_mode=False) | |