File size: 3,449 Bytes
30ea0e9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | import json
from fakes import seed_agent, seed_result
def seed_board(hub):
seed_agent(hub, "agent-1", hf_user="user-one")
seed_agent(hub, "agent-2", hf_user="user-two")
r_valid = seed_result(hub, "20260601-100000-000", "agent-1", 100.0)
r_pending = seed_result(hub, "20260602-100000-000", "agent-1", 150.0)
r2_valid = seed_result(hub, "20260601-110000-000", "agent-2", 120.0)
r2_invalid = seed_result(hub, "20260602-110000-000", "agent-2", 500.0)
seed_result(hub, "20260603-100000-000", "agent-1", 999.0, status="negative")
hub.seed(
"results/20260603-110000-000_agent-2.md", "---\nscore: [broken\n---\nmalformed"
)
hub.seed(
"results/verification_status.json",
json.dumps({r_valid: "valid", r2_valid: "valid", r2_invalid: "invalid"}),
)
return r_valid, r_pending, r2_valid, r2_invalid
def test_default_board_best_per_agent_pending_included_invalid_excluded(env):
seed_board(env.hub)
data = env.client.get("/v1/leaderboard").json()
rows = data["rows"]
assert [(r["rank"], r["agent"], r["score"], r["verification"]) for r in rows] == [
(1, "agent-1", 150.0, "pending"),
(2, "agent-2", 120.0, "valid"),
]
assert rows[0]["hf_user"] == "user-one"
assert rows[1]["method"] == "vllm-baseline"
meta = data["meta"]
assert meta["results_considered"] == 6
assert meta["excluded"] == {
"status_negative": 1,
"malformed": 1,
"verification_invalid": 1,
}
def test_strict_valid_only_view(env):
seed_board(env.hub)
rows = env.client.get("/v1/leaderboard?verification=valid").json()["rows"]
assert [(r["agent"], r["score"]) for r in rows] == [
("agent-2", 120.0),
("agent-1", 100.0),
]
def test_all_results_view(env):
seed_board(env.hub)
rows = env.client.get("/v1/leaderboard?best_per_agent=false").json()["rows"]
assert [r["score"] for r in rows] == [150.0, 120.0, 100.0]
def test_agent_filter_keeps_global_rank(env):
seed_board(env.hub)
rows = env.client.get("/v1/leaderboard?agent=agent-2").json()["rows"]
assert len(rows) == 1 and rows[0]["rank"] == 2
def test_ties_go_to_the_earlier_result(env):
seed_agent(env.hub, "agent-1")
seed_agent(env.hub, "agent-2")
seed_result(env.hub, "20260602-100000-000", "agent-2", 100.0)
seed_result(env.hub, "20260601-100000-000", "agent-1", 100.0)
rows = env.client.get("/v1/leaderboard").json()["rows"]
assert [r["agent"] for r in rows] == ["agent-1", "agent-2"]
def test_limit(env):
seed_board(env.hub)
rows = env.client.get("/v1/leaderboard?limit=1").json()["rows"]
assert len(rows) == 1 and rows[0]["rank"] == 1
def test_empty_board(env):
data = env.client.get("/v1/leaderboard").json()
assert data["rows"] == []
assert data["meta"]["results_considered"] == 0
def test_asc_order_ranks_lowest_first(make_env):
env = make_env(SCORE_ORDER="asc")
seed_agent(env.hub, "agent-1")
seed_agent(env.hub, "agent-2")
seed_result(env.hub, "20260601-100000-000", "agent-1", 100.0)
seed_result(env.hub, "20260601-110000-000", "agent-1", 80.0) # agent-1's best
seed_result(env.hub, "20260601-120000-000", "agent-2", 90.0)
data = env.client.get("/v1/leaderboard").json()
assert data["order"] == "asc"
assert [(r["agent"], r["score"]) for r in data["rows"]] == [
("agent-1", 80.0),
("agent-2", 90.0),
]
|