File size: 4,957 Bytes
730c5bb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """bankml 0.3.6 on top of PR #1: sampling options, the chat/judge models a bankml node names,
and the serve flags that layer a persona and its parameters."""
from __future__ import annotations
import asyncio
import json
import httpx
import pytest
from mindxtrain.models.registry import ChatMessage, ChatRequest
from mindxtrain.operator.backends.bankml import BankmlBackend
_ENV = ("MINDXTRAIN_CHAT_MODEL", "MINDXTRAIN_JUDGE_MODEL", "MINDXTRAIN_CHAT_OPTIONS",
"MINDXTRAIN_BANKML_OPTIONS")
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None:
for k in _ENV:
monkeypatch.delenv(k, raising=False)
def _backend(seen: list[dict[str, object]], **kw: object) -> BankmlBackend:
def handler(request: httpx.Request) -> httpx.Response:
seen.append(json.loads(request.content))
return httpx.Response(200, json={
"model": "mindx-gen39", "choices": [{"message": {"content": "I am mindX."}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 9, "completion_tokens": 4},
"bankml_receipt": {"model_sha256": "6b64c748"},
})
return BankmlBackend(transport=httpx.MockTransport(handler), **kw) # type: ignore[arg-type]
def _ask(b: BankmlBackend) -> None:
asyncio.run(b.chat(ChatRequest(model="mindx-gen39", messages=[ChatMessage(role="user", content="hi")],
max_tokens=16)))
def test_backend_sends_options_only_when_set(monkeypatch: pytest.MonkeyPatch) -> None:
seen: list[dict[str, object]] = []
_ask(_backend(seen))
assert not {"repeat_penalty", "repeat_last_n"} & set(seen[0])
monkeypatch.setenv("MINDXTRAIN_BANKML_OPTIONS", '{"repeat_penalty": 1.3, "repeat_last_n": 64}')
_ask(_backend(seen, options={"repeat_last_n": 32}))
assert seen[1]["repeat_penalty"] == 1.3 and seen[1]["repeat_last_n"] == 32 # ctor over env
assert seen[1]["model"] == "mindx-gen39" and seen[1]["max_tokens"] == 16 # request wins
def test_backend_options_must_be_an_object(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("MINDXTRAIN_BANKML_OPTIONS", "[1.3]")
with pytest.raises(ValueError, match="JSON object"):
BankmlBackend()
def test_chat_and_judge_models_follow_env(monkeypatch: pytest.MonkeyPatch) -> None:
from mindxtrain.eval.llama_evals import (
CorrectnessEvaluator,
PairwiseEvaluator,
default_judge_model,
)
from mindxtrain.governance.panel import default_chat_model
assert default_chat_model() == "llama3.2" and default_judge_model() == "llama3.2"
monkeypatch.setenv("MINDXTRAIN_CHAT_MODEL", "bonsai-8b-q1_0")
assert default_judge_model() == "bonsai-8b-q1_0" # judges follow the chat model
monkeypatch.setenv("MINDXTRAIN_JUDGE_MODEL", "bonsai-1.7b")
assert CorrectnessEvaluator().model == "bonsai-1.7b"
assert PairwiseEvaluator(model="explicit").model == "explicit"
def test_chat_once_options(monkeypatch: pytest.MonkeyPatch) -> None:
from mindxtrain.governance import panel
bodies: list[dict[str, object]] = []
def handler(request: httpx.Request) -> httpx.Response:
bodies.append(json.loads(request.content))
return httpx.Response(200, json={"choices": [{"message": {"content": "VERDICT: APPROVE"}}]})
real = httpx.Client
monkeypatch.setattr(httpx, "Client", lambda **k: real(transport=httpx.MockTransport(handler), **k))
url = "http://127.0.0.1:18093/v1"
panel.chat_once("m", [{"role": "user", "content": "x"}], base_url=url)
monkeypatch.setenv("MINDXTRAIN_CHAT_OPTIONS", '{"repeat_penalty": 1.3}')
panel.chat_once("m", [{"role": "user", "content": "x"}], base_url=url)
panel.chat_once("m", [{"role": "user", "content": "x"}], base_url=url, options={})
assert "repeat_penalty" not in bodies[0] and bodies[1]["repeat_penalty"] == 1.3
assert "repeat_penalty" not in bodies[2] # explicit {} wins
monkeypatch.setenv("MINDXTRAIN_CHAT_OPTIONS", "nope")
with pytest.raises(ValueError, match="not JSON"):
panel.chat_options()
def test_deliberate_uses_the_env_chat_model(monkeypatch: pytest.MonkeyPatch) -> None:
from mindxtrain.governance import panel
from mindxtrain.governance.boardroom import Member
asked: list[str] = []
monkeypatch.setattr(panel, "chat_once", lambda model, *a, **k: asked.append(model) or "VERDICT: APPROVE")
monkeypatch.setenv("MINDXTRAIN_CHAT_MODEL", "bonsai-8b-q1_0")
d = panel.deliberate(Member(id="m1", role="critic"), "ship it")
assert asked == ["bonsai-8b-q1_0"] and d.vote == "approve"
def test_serve_param_parsing() -> None:
from mindxtrain.cli.main import _parse_params
assert _parse_params(["repeat_penalty=1.3", "num_ctx=2048", "seed=0"]) == {
"repeat_penalty": 1.3, "num_ctx": 2048, "seed": 0}
with pytest.raises(ValueError):
_parse_params(["repeat_penalty"])
|