| import pandas as pd |
| import plotly.graph_objects as go |
|
|
| from src.analytics import ( |
| BENCHMARK_CATALOG, |
| benchmark_category, |
| benchmarks_for_category, |
| cross_benchmark_ranking_df, |
| enrich_analysis_df, |
| filter_category, |
| matrix_df, |
| ranking_df, |
| ) |
| from src.charts import create_coverage_matrix_plot, create_matrix_plot, create_tradeoff_plot |
|
|
|
|
| def frame(): |
| return pd.DataFrame( |
| [ |
| { |
| "Benchmark": "SWE-Bench Verified", "Model": "a", "Harness": "h", "Run Label": "a / h", |
| "Category": "FOSS", "Score": .8, "Score (%)": 80, "Tasks": 10, "Errors": 1, |
| "Input Tokens Per Task": 50, "Cache Tokens Per Task": 10, "Output Tokens Per Task": 20, |
| "Total Tokens Per Task": 80, "Tokens Per Solved Task": 100, "Cost Per Task": .2, |
| "Total Time Per Task": 10, "Agent Time Per Task": 8, "Token Data Available": True, |
| }, |
| { |
| "Benchmark": "SWE-Bench Verified", "Model": "b", "Harness": "h", "Run Label": "b / h", |
| "Category": "FOSS", "Score": .4, "Score (%)": 40, "Tasks": 10, "Errors": 2, |
| "Input Tokens Per Task": 90, "Cache Tokens Per Task": None, "Output Tokens Per Task": 30, |
| "Total Tokens Per Task": 120, "Tokens Per Solved Task": 300, "Cost Per Task": .1, |
| "Total Time Per Task": 20, "Agent Time Per Task": 15, "Token Data Available": True, |
| }, |
| { |
| "Benchmark": "Terminal Bench 2.0", "Model": "a", "Harness": "h", "Run Label": "a / h", |
| "Category": "FOSS", "Score": .2, "Score (%)": 20, "Tasks": 10, "Errors": 0, |
| "Input Tokens Per Task": 30, "Cache Tokens Per Task": 5, "Output Tokens Per Task": 10, |
| "Total Tokens Per Task": 45, "Tokens Per Solved Task": 225, "Cost Per Task": .3, |
| "Total Time Per Task": 30, "Agent Time Per Task": 25, "Token Data Available": True, |
| }, |
| { |
| "Benchmark": "Terminal Bench 2.0", "Model": "c", "Harness": "h", "Run Label": "c / h", |
| "Category": "FOSS", "Score": .9, "Score (%)": 90, "Tasks": 10, "Errors": None, |
| "Input Tokens Per Task": None, "Cache Tokens Per Task": None, "Output Tokens Per Task": None, |
| "Total Tokens Per Task": None, "Tokens Per Solved Task": None, "Cost Per Task": None, |
| "Total Time Per Task": None, "Agent Time Per Task": None, "Token Data Available": False, |
| }, |
| { |
| "Benchmark": "New Benchmark", "Model": "z", "Harness": "h", "Run Label": "z / h", |
| "Category": "FOSS", "Score": .5, "Score (%)": 50, "Tasks": 10, "Errors": 0, |
| "Input Tokens Per Task": 1, "Cache Tokens Per Task": 1, "Output Tokens Per Task": 1, |
| "Total Tokens Per Task": 3, "Tokens Per Solved Task": 6, "Cost Per Task": .01, |
| "Total Time Per Task": 1, "Agent Time Per Task": 1, "Token Data Available": True, |
| }, |
| ] |
| ) |
|
|
|
|
| def test_benchmark_catalog_and_unknown_fallback(): |
| assert benchmark_category("SWE-Bench Verified") == "Coding" |
| assert benchmark_category("Terminal Bench 2.0") == "Generalist" |
| assert benchmark_category("Shellbench") == "Generalist" |
| assert benchmark_category("ITBench AA") == "SRE" |
| assert benchmark_category("New Benchmark") == "Other" |
| assert "SWE-Bench Pro -- Ansible" in BENCHMARK_CATALOG |
| assert "ITBench AA" in BENCHMARK_CATALOG |
|
|
|
|
| def test_category_filtering_keeps_unknown_visible_as_other(): |
| df = frame() |
| assert set(benchmarks_for_category(df, "Coding")) == {"SWE-Bench Verified"} |
| assert set(filter_category(df, "Generalist")["Benchmark"]) == {"Terminal Bench 2.0"} |
| assert set(filter_category(df, "Other")["Benchmark"]) == {"New Benchmark"} |
|
|
|
|
| def test_derived_reliability_and_per_success_metrics(): |
| df = enrich_analysis_df(frame()) |
| first = df.iloc[0] |
| assert first["Execution Error Rate (%)"] == 10 |
| assert first["Tokens Per Successful Task"] == 100 |
| assert first["Cost Per Successful Task"] == .25 |
| assert first["Time Per Successful Task"] == 12.5 |
| assert pd.isna(df.loc[df["Model"].eq("c"), "Execution Error Rate (%)"]).all() |
|
|
|
|
| def test_within_benchmark_percentile_and_rank(): |
| df = enrich_analysis_df(frame()) |
| coding = df[df["Benchmark"] == "SWE-Bench Verified"].set_index("Model") |
| assert coding.loc["a", "Within-Benchmark Rank"] == 1 |
| assert coding.loc["b", "Within-Benchmark Rank"] == 2 |
| assert coding.loc["a", "Within-Benchmark Percentile"] == 100 |
| assert coding.loc["b", "Within-Benchmark Percentile"] == 0 |
|
|
|
|
| def test_cross_benchmark_ordering_uses_normalization_and_coverage_threshold(): |
| df = frame() |
| ranked = cross_benchmark_ranking_df(df, minimum_coverage=0.5) |
| a = ranked[ranked["Model"] == "a"].iloc[0] |
| assert a["Benchmarks Covered"] == 2 |
| assert a["Eligible Benchmarks"] == 3 |
| assert a["Normalized Performance"] == 50 |
| strict = cross_benchmark_ranking_df(df, minimum_coverage=1.0) |
| assert strict.empty |
|
|
|
|
| def test_token_ranking_excludes_missing_and_orders_lower_first(): |
| ranked = ranking_df(frame(), "Total tokens", benchmark="Terminal Bench 2.0") |
| assert ranked["Model"].tolist() == ["a"] |
|
|
|
|
| def test_score_rank_and_metric_matrices_preserve_missing_cells(): |
| df = frame() |
| score = matrix_df(df, "Score") |
| rank = matrix_df(df, "Within-benchmark rank") |
| cost = matrix_df(df, "Cost") |
| assert pd.isna(score.loc["b / h", "Terminal Bench 2.0"]) |
| assert rank.loc["a / h", "SWE-Bench Verified"] == 1 |
| assert pd.isna(cost.loc["c / h", "Terminal Bench 2.0"]) |
|
|
|
|
| def test_coverage_matrix_uses_missing_not_zero_score(): |
| matrix = matrix_df(frame(), "Coverage") |
| assert matrix.loc["a / h", "SWE-Bench Verified"] == 1 |
| assert pd.isna(matrix.loc["b / h", "Terminal Bench 2.0"]) |
| figure = create_coverage_matrix_plot(matrix) |
| assert isinstance(figure, go.Figure) |
| assert "Available" in figure.data[0].text[0] or "Missing" in figure.data[0].text[0] |
|
|
|
|
| def test_generic_tradeoff_and_matrix_charts_construct(): |
| df = enrich_analysis_df(frame()) |
| figure = create_tradeoff_plot( |
| df[df["Benchmark"] == "SWE-Bench Verified"], |
| "Total Tokens Per Task", "Score (%)", |
| "Total tokens per task", "Score (%)", |
| show_pareto_frontier=True, |
| ) |
| matrix_figure = create_matrix_plot(matrix_df(df, "Score"), "Score matrix", "Score (%)") |
| assert isinstance(figure, go.Figure) |
| assert isinstance(matrix_figure, go.Figure) |
| assert any(trace.name == "Pareto frontier" for trace in figure.data) |
|
|
|
|
| def test_tradeoff_pareto_supports_lower_is_better_on_both_axes(): |
| df = pd.DataFrame( |
| { |
| "Model": ["a", "b", "c"], |
| "Harness": ["h", "h", "h"], |
| "Benchmark": ["bench", "bench", "bench"], |
| "Total Tokens Per Task": [100, 200, 300], |
| "Cost Per Task": [0.3, 0.2, 0.4], |
| } |
| ) |
| figure = create_tradeoff_plot( |
| df, |
| "Total Tokens Per Task", |
| "Cost Per Task", |
| "Tokens", |
| "Cost", |
| show_pareto_frontier=True, |
| lower_x_is_better=True, |
| higher_y_is_better=False, |
| ) |
| frontier = next(trace for trace in figure.data if trace.name == "Pareto frontier") |
| assert list(frontier.x) == [100, 200] |
| assert list(frontier.y) == [0.3, 0.2] |
|
|
|
|
| def test_matrix_can_color_by_percentile_but_display_raw_values(): |
| color_matrix = pd.DataFrame([[100.0, 0.0]], index=["a / h"], columns=["b1", "b2"] ) |
| raw_matrix = pd.DataFrame([[82.5, 41.25]], index=["a / h"], columns=["b1", "b2"] ) |
| figure = create_matrix_plot( |
| color_matrix, |
| "Score matrix", |
| "Within-benchmark percentile", |
| display_matrix=raw_matrix, |
| display_metric_label="Benchmark score (%)", |
| ) |
| assert list(figure.data[0].z[0]) == [100.0, 0.0] |
| assert list(figure.data[0].text[0]) == ["82.5", "41.25"] |
| assert "Benchmark score (%)" in figure.data[0].hovertemplate |
|
|
|
|
| def test_ranking_plot_order_can_be_value_or_alphabetical(): |
| from src.charts import create_ranking_plot |
|
|
| df = pd.DataFrame( |
| { |
| "Model": ["b", "a", "c"], |
| "Harness": ["h", "h", "h"], |
| "Benchmark": ["bench", "bench", "bench"], |
| "Score (%)": [20, 10, 30], |
| } |
| ) |
| largest = create_ranking_plot( |
| df, "Score (%)", "Score", True, sort_order="Largest value first" |
| ) |
| lowest = create_ranking_plot( |
| df, "Score (%)", "Score", True, sort_order="Lowest value first" |
| ) |
| alpha = create_ranking_plot( |
| df, "Score (%)", "Score", True, sort_order="Alphabetical (A–Z)" |
| ) |
|
|
| assert list(largest.layout.yaxis.categoryarray) == ["c / h", "b / h", "a / h"] |
| assert list(lowest.layout.yaxis.categoryarray) == ["a / h", "b / h", "c / h"] |
| assert list(alpha.layout.yaxis.categoryarray) == ["a / h", "b / h", "c / h"] |
|
|
|
|
| def test_matrix_height_scales_with_rows_and_keeps_all_y_labels(): |
| rows = 18 |
| matrix = pd.DataFrame( |
| {"bench-a": range(rows), "bench-b": range(rows)}, |
| index=[f"model-{i} / harness" for i in range(rows)], |
| ) |
|
|
| figure = create_matrix_plot(matrix, "Dense matrix", "Score (%)") |
|
|
| assert figure.layout.height >= 900 |
| assert figure.layout.yaxis.tickmode == "array" |
| assert len(figure.layout.yaxis.tickvals) == rows |
| assert len(figure.layout.yaxis.ticktext) == rows |
|
|
