CP Legendre commited on
Commit ·
aa954a8
1
Parent(s): fedecbd
test: use eligible benchmarks in harness ranking fixture
Browse files
tests/test_ranking_refinements.py
CHANGED
|
@@ -107,12 +107,10 @@ def test_ranking_plot_height_scales_with_agents_and_keeps_all_y_labels():
|
|
| 107 |
def test_pairwise_harness_rankings_exclude_low_data_harnesses(tmp_path):
|
| 108 |
rankings = _rankings_module()
|
| 109 |
csv_path = tmp_path / "results.csv"
|
|
|
|
| 110 |
pd.DataFrame(
|
| 111 |
{
|
| 112 |
-
"benchmark.name": [
|
| 113 |
-
"Bench A", "Bench A", "Bench A", "Bench A",
|
| 114 |
-
"Bench B", "Bench B", "Bench B", "Bench B",
|
| 115 |
-
],
|
| 116 |
"model.name": ["m1", "m1", "m1", "m1", "m2", "m2", "m2", "m2"],
|
| 117 |
"harness.name": [
|
| 118 |
"Claude Code", "OpenCode", "Codex", "Qwen Code",
|
|
@@ -136,7 +134,7 @@ def test_pairwise_harness_rankings_exclude_low_data_harnesses(tmp_path):
|
|
| 136 |
assert set(model_df["Category"]) == {"m1", "m2"}
|
| 137 |
|
| 138 |
model_by_category = model_df.set_index("Category")
|
| 139 |
-
assert model_by_category.loc["m1", "
|
| 140 |
|
| 141 |
|
| 142 |
def test_ranking_minimum_height_is_reduced_without_changing_matrix_minimum():
|
|
|
|
| 107 |
def test_pairwise_harness_rankings_exclude_low_data_harnesses(tmp_path):
|
| 108 |
rankings = _rankings_module()
|
| 109 |
csv_path = tmp_path / "results.csv"
|
| 110 |
+
# Use eligible benchmarks so this exercises harness exclusions, not the benchmark allowlist.
|
| 111 |
pd.DataFrame(
|
| 112 |
{
|
| 113 |
+
"benchmark.name": ["RH SWE-Bench"] * 4 + ["SWE-Bench Verified"] * 4,
|
|
|
|
|
|
|
|
|
|
| 114 |
"model.name": ["m1", "m1", "m1", "m1", "m2", "m2", "m2", "m2"],
|
| 115 |
"harness.name": [
|
| 116 |
"Claude Code", "OpenCode", "Codex", "Qwen Code",
|
|
|
|
| 134 |
assert set(model_df["Category"]) == {"m1", "m2"}
|
| 135 |
|
| 136 |
model_by_category = model_df.set_index("Category")
|
| 137 |
+
assert model_by_category.loc["m1", "RH SWE-Bench"] == pytest.approx(0.838)
|
| 138 |
|
| 139 |
|
| 140 |
def test_ranking_minimum_height_is_reduced_without_changing_matrix_minimum():
|