CP Legendre commited on
Commit
aa954a8
·
1 Parent(s): fedecbd

test: use eligible benchmarks in harness ranking fixture

Browse files
Files changed (1) hide show
  1. tests/test_ranking_refinements.py +3 -5
tests/test_ranking_refinements.py CHANGED
@@ -107,12 +107,10 @@ def test_ranking_plot_height_scales_with_agents_and_keeps_all_y_labels():
107
  def test_pairwise_harness_rankings_exclude_low_data_harnesses(tmp_path):
108
  rankings = _rankings_module()
109
  csv_path = tmp_path / "results.csv"
 
110
  pd.DataFrame(
111
  {
112
- "benchmark.name": [
113
- "Bench A", "Bench A", "Bench A", "Bench A",
114
- "Bench B", "Bench B", "Bench B", "Bench B",
115
- ],
116
  "model.name": ["m1", "m1", "m1", "m1", "m2", "m2", "m2", "m2"],
117
  "harness.name": [
118
  "Claude Code", "OpenCode", "Codex", "Qwen Code",
@@ -136,7 +134,7 @@ def test_pairwise_harness_rankings_exclude_low_data_harnesses(tmp_path):
136
  assert set(model_df["Category"]) == {"m1", "m2"}
137
 
138
  model_by_category = model_df.set_index("Category")
139
- assert model_by_category.loc["m1", "Bench A"] == pytest.approx(0.838)
140
 
141
 
142
  def test_ranking_minimum_height_is_reduced_without_changing_matrix_minimum():
 
107
  def test_pairwise_harness_rankings_exclude_low_data_harnesses(tmp_path):
108
  rankings = _rankings_module()
109
  csv_path = tmp_path / "results.csv"
110
+ # Use eligible benchmarks so this exercises harness exclusions, not the benchmark allowlist.
111
  pd.DataFrame(
112
  {
113
+ "benchmark.name": ["RH SWE-Bench"] * 4 + ["SWE-Bench Verified"] * 4,
 
 
 
114
  "model.name": ["m1", "m1", "m1", "m1", "m2", "m2", "m2", "m2"],
115
  "harness.name": [
116
  "Claude Code", "OpenCode", "Codex", "Qwen Code",
 
134
  assert set(model_df["Category"]) == {"m1", "m2"}
135
 
136
  model_by_category = model_df.set_index("Category")
137
+ assert model_by_category.loc["m1", "RH SWE-Bench"] == pytest.approx(0.838)
138
 
139
 
140
  def test_ranking_minimum_height_is_reduced_without_changing_matrix_minimum():