Spaces:
Running
Running
Download source/tests/test_bench_fastest.py from munip/collusion-watch: direct link, hf CLI and curl.
- Browser
- Download file 2.42 kB
-
https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/tests/test_bench_fastest.py
- Command line
-
hf download hf://spaces/munip/collusion-watch/source/tests/test_bench_fastest.py
-
curl -L -o test_bench_fastest.py https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/tests/test_bench_fastest.py
2.42 kB
| import json | |
| import time | |
| from collusion_lab.core import bench as B | |
| from collusion_lab.core import pool as P | |
| def test_pick_fastest_requires_reliability(): | |
| rows = [{"spec": "a/fast-but-rambles", "median_s": 0.5, "fail_rate": 0.0, "json_ok": 0.33, "tool_ok": 1.0, "score": 20.5}, | |
| {"spec": "b/fast-but-429", "median_s": 0.4, "fail_rate": 0.5, "json_ok": 1.0, "tool_ok": 1.0, "score": 0.8}, | |
| {"spec": "c/steady", "median_s": 3.0, "fail_rate": 0.0, "json_ok": 1.0, "tool_ok": 1.0, "score": 3.0}, | |
| {"spec": "d/slow", "median_s": 9.0, "fail_rate": 0.0, "json_ok": 1.0, "tool_ok": 1.0, "score": 9.0}] | |
| assert B.pick_fastest(rows)["spec"] == "c/steady" | |
| assert B.pick_fastest(rows[:2]) is None | |
| def test_bench_scores_and_caches(monkeypatch, tmp_path): | |
| monkeypatch.setattr(B, "BENCH_FILE", tmp_path / "b.json") | |
| monkeypatch.setattr(B, "get_inspect_model", lambda r: r.spec) | |
| async def fake_call(model, kind, timeout): | |
| if model == "groq/flaky": | |
| return (None, False, "RateLimitError") if kind == "tool" else (0.2, True, "") | |
| return (1.0 if model == "groq/quick" else 4.0), True, "" | |
| monkeypatch.setattr(B, "_call", fake_call) | |
| rows = B.bench(["groq/slow", "groq/quick", "groq/flaky"], n=4) | |
| assert [r["spec"] for r in rows][0] == "groq/quick" | |
| assert B.pick_fastest(rows)["spec"] == "groq/quick" | |
| assert B.cached()[0]["spec"] == "groq/quick" | |
| def test_auto_fastest_pins_every_agent_to_one_model_and_judge_to_another_family(monkeypatch, tmp_path): | |
| monkeypatch.setattr(B, "BENCH_FILE", tmp_path / "b.json") | |
| (tmp_path / "b.json").write_text(json.dumps({"created": time.time(), "results": [ | |
| {"spec": "cerebras/qwen-3.8-27b", "family": "qwen", "median_s": 1.2, "fail_rate": 0.0, "json_ok": 1.0, | |
| "tool_ok": 1.0, "score": 1.2}]})) | |
| pool = [P.Candidate("cerebras/qwen-3.8-27b", "cerebras", "qwen-3.8-27b", "qwen", "x", ok=True, tools="native", score=5), | |
| P.Candidate("groq/openai/gpt-oss-120b", "groq", "openai/gpt-oss-120b", "gpt", "x", ok=True, tools="native", score=4)] | |
| monkeypatch.setattr(P, "discover", lambda *a, **k: pool) | |
| out = P.expand_auto({"default": "auto:fastest", "monitor_judge": "auto:judge", "qa_peer": "auto:judge"}, {}) | |
| assert out["implementer"] == out["seller"] == ["cerebras/qwen-3.8-27b"] | |
| assert out["monitor_judge"] == out["qa_peer"] == ["groq/openai/gpt-oss-120b"] | |