collusion-watch / source /tests /test_providers.py
munip's picture
Publish Collusion-Watch article, source, and reviewed results
bd26047 verified
Raw History Blame Contribute Delete
6.57 kB
"""Multi-provider plumbing: spec resolution, family separation, fallback chains, JSON tool mode."""
import pytest
from inspect_ai.model import ModelOutput, ModelUsage, get_model
from collusion_lab.core.agent_loop import ToolSpec, parse_json_call, run_sync, tool_loop
from collusion_lab.core.models import AllModelsFailed, ModelRoster
from collusion_lab.core.providers import family, resolve
def _out(text=None, tool=None, args=None):
o = (ModelOutput.for_tool_call("mockllm/model", tool, args or {}) if tool
else ModelOutput.from_content("mockllm/model", text))
o.usage = ModelUsage(input_tokens=5, output_tokens=5, total_tokens=10)
return o
def test_resolve_openai_compatible_and_router(monkeypatch):
monkeypatch.setenv("GROQ_API_KEY", "gsk_test_123456")
r = resolve("groq/openai/gpt-oss-120b")
assert r.kind == "openai_compat" and r.inspect_name == "openai-api/groq/openai/gpt-oss-120b"
assert r.base_url.startswith("https://api.groq.com") and r.has_key and r.family == "gpt"
f = resolve("fcm/fcm:agents")
assert f.kind == "router" and f.base_url.endswith(":19280/v1") and f.family == "router:dynamic"
o = resolve("ollama/gemma2:2b")
assert o.kind == "native" and o.family == "gemma" and o.tools == "auto"
p = resolve("puter/openai/gpt-5.4-nano")
assert p.kind == "openai_compat" and p.base_url == "http://127.0.0.1:3800/v1"
assert not p.needs_key
monkeypatch.setenv("OLLAMA_BASE_URL", "http://192.168.1.5:11434/v1")
assert resolve("ollama/gemma2:2b").base_url == "http://192.168.1.5:11434/v1"
@pytest.mark.parametrize("model,fam", [("llama-3.3-70b-versatile", "llama"), ("qwen/qwen3-coder:free", "qwen"),
("gemini-2.5-flash", "gemini"), ("moonshotai/kimi-k2-instruct", "kimi"),
("claude-haiku-4-5", "claude"), ("o3-mini", "gpt"), ("mixtral-8x7b", "mistral")])
def test_family(model, fam):
assert family(model) == fam
def test_profiles_keep_monitor_family_separate():
for prof in ("free", "free_homogeneous", "free_weak_monitor", "free_gpt_oss20b_failover",
"local", "cloud", "mixed"):
r = ModelRoster(prof)
assert not any("shares model family" in w for w in r.check_separation()), prof
mom = ModelRoster("puter_live_mom")
assert mom.chain("monitor_auditor") == ["puter/openai/gpt-5.4-mini"]
assert mom.chain("monitor_judge") == ["puter/anthropic/claude-sonnet-5-5"]
assert not any("different model spec" in w for w in mom.check_separation())
assert any("different model spec" in w for w in ModelRoster(
"scripted", {"monitor_judge": "mockllm/same", "monitor_auditor": "mockllm/same"}
).check_separation())
assert any("shares model family" in w for w in ModelRoster("same_family", {}).check_separation()) is False # allowed ablation
bad = ModelRoster("free", {"monitor_judge": "groq/llama-3.3-70b-versatile"})
assert any("shares model family" in w for w in bad.check_separation())
class Boom:
async def generate(self, *a, **k):
raise RuntimeError("temporary failure")
def test_fallback_chain_uses_next_model():
roster = ModelRoster("scripted", {"implementer": "mockllm/a,mockllm/b"})
roster._models["mockllm/a"] = Boom()
roster._models["mockllm/b"] = get_model("mockllm/model", custom_outputs=[_out("hello")])
out = run_sync(roster.generate("implementer", []))
assert out.metadata["clab_spec"] == "mockllm/b" and roster.meter.failures == {"mockllm/a": 1}
roster2 = ModelRoster("scripted", {"implementer": "mockllm/a"})
roster2._models["mockllm/a"] = Boom()
with pytest.raises(AllModelsFailed):
run_sync(roster2.generate("implementer", []))
def test_rate_limit_fails_over_and_cools_down_provider(monkeypatch):
from inspect_ai.model import ChatMessageUser
from collusion_lab.core.models import RateLimiter
monkeypatch.setenv("GROQ_API_KEY", "gsk_test_123456")
monkeypatch.setenv("NVIDIA_API_KEY", "nvapi_test_123456")
monkeypatch.setattr(RateLimiter, "reserve", classmethod(lambda cls, provider, rpm: 0.0))
roster = ModelRoster("scripted", {
"implementer": "groq/openai/gpt-oss-20b,nvidia/openai/gpt-oss-20b",
})
groq_calls = []
nvidia_calls = []
retries = []
class RateLimited:
async def generate(self, *a, **k):
groq_calls.append(1)
retries.append(k["config"].max_retries)
raise RuntimeError("429 rate limit")
class Available:
async def generate(self, *a, **k):
nvidia_calls.append(1)
retries.append(k["config"].max_retries)
return _out("ok")
monkeypatch.setattr(roster, "_get", lambda r: RateLimited() if r.provider == "groq" else Available())
first = run_sync(roster.generate("implementer", [ChatMessageUser(content="hi")]))
second = run_sync(roster.generate("implementer", [ChatMessageUser(content="hi")]))
assert first.metadata["clab_spec"] == second.metadata["clab_spec"] == "nvidia/openai/gpt-oss-20b"
assert groq_calls == [1]
assert nvidia_calls == [1, 1]
assert retries == [0, 0, 0]
assert roster.meter.failures == {"groq/openai/gpt-oss-20b": 1}
def test_parse_json_call_variants():
names = {"read_file", "done"}
assert parse_json_call('```json\n{"tool": "read_file", "args": {"path": "SPEC.md"}}\n```', names) == ("read_file", {"path": "SPEC.md"})
assert parse_json_call('I will finish. {"name": "done", "arguments": "{}"}', names) == ("done", {})
assert parse_json_call("no json here", names) is None
def test_json_tool_mode_loop(monkeypatch):
"""A model without function calling (e.g. gemma2 on Ollama) drives tools via JSON text."""
roster = ModelRoster("scripted", {"implementer": "mockllm/json"})
monkeypatch.setattr(roster, "tool_mode", lambda role, spec=None: "json")
roster._models["mockllm/json"] = get_model("mockllm/model", custom_outputs=[
_out('{"tool": "send_message", "args": {"to": "tester", "text": "hi"}}'),
_out("thinking..."),
_out('{"tool": "done", "args": {}}')])
seen = []
tools = [ToolSpec("send_message", "msg", {"to": ("string", ""), "text": ("string", "")}, ("to", "text")),
ToolSpec("done", "end", {})]
msgs = run_sync(tool_loop(roster, "implementer", "sys", "go", tools, lambda n, a: seen.append((n, a)) or "ok", 6))
assert seen == [("send_message", {"to": "tester", "text": "hi"}), ("done", {})]
assert "TOOLS (call exactly one per reply)" in msgs[0].text