Spaces:
Running
Running
Download source/tests/test_providers.py from munip/collusion-watch: direct link, hf CLI and curl.
- Browser
- Download file 6.57 kB
-
https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/tests/test_providers.py
- Command line
-
hf download hf://spaces/munip/collusion-watch/source/tests/test_providers.py
-
curl -L -o test_providers.py https://huggingface.co/spaces/munip/collusion-watch/resolve/main/source/tests/test_providers.py
6.57 kB
| """Multi-provider plumbing: spec resolution, family separation, fallback chains, JSON tool mode.""" | |
| import pytest | |
| from inspect_ai.model import ModelOutput, ModelUsage, get_model | |
| from collusion_lab.core.agent_loop import ToolSpec, parse_json_call, run_sync, tool_loop | |
| from collusion_lab.core.models import AllModelsFailed, ModelRoster | |
| from collusion_lab.core.providers import family, resolve | |
| def _out(text=None, tool=None, args=None): | |
| o = (ModelOutput.for_tool_call("mockllm/model", tool, args or {}) if tool | |
| else ModelOutput.from_content("mockllm/model", text)) | |
| o.usage = ModelUsage(input_tokens=5, output_tokens=5, total_tokens=10) | |
| return o | |
| def test_resolve_openai_compatible_and_router(monkeypatch): | |
| monkeypatch.setenv("GROQ_API_KEY", "gsk_test_123456") | |
| r = resolve("groq/openai/gpt-oss-120b") | |
| assert r.kind == "openai_compat" and r.inspect_name == "openai-api/groq/openai/gpt-oss-120b" | |
| assert r.base_url.startswith("https://api.groq.com") and r.has_key and r.family == "gpt" | |
| f = resolve("fcm/fcm:agents") | |
| assert f.kind == "router" and f.base_url.endswith(":19280/v1") and f.family == "router:dynamic" | |
| o = resolve("ollama/gemma2:2b") | |
| assert o.kind == "native" and o.family == "gemma" and o.tools == "auto" | |
| p = resolve("puter/openai/gpt-5.4-nano") | |
| assert p.kind == "openai_compat" and p.base_url == "http://127.0.0.1:3800/v1" | |
| assert not p.needs_key | |
| monkeypatch.setenv("OLLAMA_BASE_URL", "http://192.168.1.5:11434/v1") | |
| assert resolve("ollama/gemma2:2b").base_url == "http://192.168.1.5:11434/v1" | |
| def test_family(model, fam): | |
| assert family(model) == fam | |
| def test_profiles_keep_monitor_family_separate(): | |
| for prof in ("free", "free_homogeneous", "free_weak_monitor", "free_gpt_oss20b_failover", | |
| "local", "cloud", "mixed"): | |
| r = ModelRoster(prof) | |
| assert not any("shares model family" in w for w in r.check_separation()), prof | |
| mom = ModelRoster("puter_live_mom") | |
| assert mom.chain("monitor_auditor") == ["puter/openai/gpt-5.4-mini"] | |
| assert mom.chain("monitor_judge") == ["puter/anthropic/claude-sonnet-5-5"] | |
| assert not any("different model spec" in w for w in mom.check_separation()) | |
| assert any("different model spec" in w for w in ModelRoster( | |
| "scripted", {"monitor_judge": "mockllm/same", "monitor_auditor": "mockllm/same"} | |
| ).check_separation()) | |
| assert any("shares model family" in w for w in ModelRoster("same_family", {}).check_separation()) is False # allowed ablation | |
| bad = ModelRoster("free", {"monitor_judge": "groq/llama-3.3-70b-versatile"}) | |
| assert any("shares model family" in w for w in bad.check_separation()) | |
| class Boom: | |
| async def generate(self, *a, **k): | |
| raise RuntimeError("temporary failure") | |
| def test_fallback_chain_uses_next_model(): | |
| roster = ModelRoster("scripted", {"implementer": "mockllm/a,mockllm/b"}) | |
| roster._models["mockllm/a"] = Boom() | |
| roster._models["mockllm/b"] = get_model("mockllm/model", custom_outputs=[_out("hello")]) | |
| out = run_sync(roster.generate("implementer", [])) | |
| assert out.metadata["clab_spec"] == "mockllm/b" and roster.meter.failures == {"mockllm/a": 1} | |
| roster2 = ModelRoster("scripted", {"implementer": "mockllm/a"}) | |
| roster2._models["mockllm/a"] = Boom() | |
| with pytest.raises(AllModelsFailed): | |
| run_sync(roster2.generate("implementer", [])) | |
| def test_rate_limit_fails_over_and_cools_down_provider(monkeypatch): | |
| from inspect_ai.model import ChatMessageUser | |
| from collusion_lab.core.models import RateLimiter | |
| monkeypatch.setenv("GROQ_API_KEY", "gsk_test_123456") | |
| monkeypatch.setenv("NVIDIA_API_KEY", "nvapi_test_123456") | |
| monkeypatch.setattr(RateLimiter, "reserve", classmethod(lambda cls, provider, rpm: 0.0)) | |
| roster = ModelRoster("scripted", { | |
| "implementer": "groq/openai/gpt-oss-20b,nvidia/openai/gpt-oss-20b", | |
| }) | |
| groq_calls = [] | |
| nvidia_calls = [] | |
| retries = [] | |
| class RateLimited: | |
| async def generate(self, *a, **k): | |
| groq_calls.append(1) | |
| retries.append(k["config"].max_retries) | |
| raise RuntimeError("429 rate limit") | |
| class Available: | |
| async def generate(self, *a, **k): | |
| nvidia_calls.append(1) | |
| retries.append(k["config"].max_retries) | |
| return _out("ok") | |
| monkeypatch.setattr(roster, "_get", lambda r: RateLimited() if r.provider == "groq" else Available()) | |
| first = run_sync(roster.generate("implementer", [ChatMessageUser(content="hi")])) | |
| second = run_sync(roster.generate("implementer", [ChatMessageUser(content="hi")])) | |
| assert first.metadata["clab_spec"] == second.metadata["clab_spec"] == "nvidia/openai/gpt-oss-20b" | |
| assert groq_calls == [1] | |
| assert nvidia_calls == [1, 1] | |
| assert retries == [0, 0, 0] | |
| assert roster.meter.failures == {"groq/openai/gpt-oss-20b": 1} | |
| def test_parse_json_call_variants(): | |
| names = {"read_file", "done"} | |
| assert parse_json_call('```json\n{"tool": "read_file", "args": {"path": "SPEC.md"}}\n```', names) == ("read_file", {"path": "SPEC.md"}) | |
| assert parse_json_call('I will finish. {"name": "done", "arguments": "{}"}', names) == ("done", {}) | |
| assert parse_json_call("no json here", names) is None | |
| def test_json_tool_mode_loop(monkeypatch): | |
| """A model without function calling (e.g. gemma2 on Ollama) drives tools via JSON text.""" | |
| roster = ModelRoster("scripted", {"implementer": "mockllm/json"}) | |
| monkeypatch.setattr(roster, "tool_mode", lambda role, spec=None: "json") | |
| roster._models["mockllm/json"] = get_model("mockllm/model", custom_outputs=[ | |
| _out('{"tool": "send_message", "args": {"to": "tester", "text": "hi"}}'), | |
| _out("thinking..."), | |
| _out('{"tool": "done", "args": {}}')]) | |
| seen = [] | |
| tools = [ToolSpec("send_message", "msg", {"to": ("string", ""), "text": ("string", "")}, ("to", "text")), | |
| ToolSpec("done", "end", {})] | |
| msgs = run_sync(tool_loop(roster, "implementer", "sys", "go", tools, lambda n, a: seen.append((n, a)) or "ok", 6)) | |
| assert seen == [("send_message", {"to": "tester", "text": "hi"}), ("done", {})] | |
| assert "TOOLS (call exactly one per reply)" in msgs[0].text | |