| """Imprint probes through bankml: decoding sent, receipts kept, refusals typed, never mixed with |
| the canonical gate. MockTransport only — no server, no torch.""" |
|
|
| from __future__ import annotations |
|
|
| import json |
|
|
| import httpx |
| import pytest |
|
|
| from mindxtrain.eval.imprint_bankml import ( |
| NOT_COMPARABLE_NOTE, |
| imprint_via_bankml, |
| probe_recall_via_bankml, |
| ) |
| from mindxtrain.operator.backends.bankml import BankmlRefusal |
|
|
| BASE_SHA = "1" * 64 |
| GEN_SHA = "2" * 64 |
| VOICE = ["I am mindX, the sovereign workshop of the professor."] |
| ANSWERS = { |
| "smollm2-base": ("Hello, how can I help you today?", BASE_SHA), |
| "mindx-gen99": ("I am mindX, the sovereign workshop.", GEN_SHA), |
| } |
|
|
|
|
| def _transport(seen: list[dict[str, object]]) -> httpx.MockTransport: |
| def handler(request: httpx.Request) -> httpx.Response: |
| assert request.url.path == "/api/chat" |
| body = json.loads(request.content) |
| seen.append(body) |
| text, sha = ANSWERS[body["model"]] |
| return httpx.Response(200, json={ |
| "model": body["model"], "message": {"role": "assistant", "content": text}, |
| "done": True, "done_reason": "stop", "eval_count": 9, |
| "bankml_receipt": {"model_sha256": sha, "request_sha256": "r" * 64, |
| "response_sha256": "s" * 64, "engine": "native"}, |
| }) |
|
|
| return httpx.MockTransport(handler) |
|
|
|
|
| def test_probe_sends_greedy_seeded_unpenalised_decoding() -> None: |
| seen: list[dict[str, object]] = [] |
| probe = probe_recall_via_bankml( |
| "mindx-gen99", ["Who are you?", "Say hello."], system="You are mindX.", seed=42, |
| transport=_transport(seen), |
| ) |
| assert probe.utterances == ["I am mindX, the sovereign workshop."] * 2 |
| assert probe.model_sha256 == [GEN_SHA] |
| assert all(r and r["model_sha256"] == GEN_SHA for r in probe.receipts) |
| for body in seen: |
| assert body["stream"] is False |
| assert body["options"] == {"temperature": 0.0, "seed": 42, "num_predict": 48} |
| msgs = body["messages"] |
| assert isinstance(msgs, list) and msgs[0] == {"role": "system", "content": "You are mindX."} |
|
|
|
|
| def test_imprint_report_is_tagged_and_not_comparable() -> None: |
| res = imprint_via_bankml("smollm2-base", "mindx-gen99", ["Who are you?"], VOICE, |
| seed=3, transport=_transport([])) |
| assert res.report.method.endswith("/bankml-greedy") |
| assert res.report.imprint_delta > 0 and res.report.imprinted |
| assert res.canonical_gate is False |
| assert res.note == NOT_COMPARABLE_NOTE and "NOT comparable" in res.note |
| assert res.decoding.seed == 3 and res.decoding.temperature == 0.0 |
| assert res.before.model_sha256 == [BASE_SHA] and res.after.model_sha256 == [GEN_SHA] |
| |
| assert json.loads(res.model_dump_json())["comparable_with"] == "bankml-greedy only" |
|
|
|
|
| def test_refusal_is_raised_typed_and_not_retried() -> None: |
| calls = {"n": 0} |
|
|
| def handler(request: httpx.Request) -> httpx.Response: |
| calls["n"] += 1 |
| return httpx.Response(400, json={"error": "options.weird: not an option bankML knows"}) |
|
|
| with pytest.raises(BankmlRefusal, match="not an option bankML knows"): |
| probe_recall_via_bankml("x", ["a", "b"], transport=httpx.MockTransport(handler)) |
| assert calls["n"] == 1 |
|
|
|
|
| def test_base_url_root_is_derived_from_v1(monkeypatch: pytest.MonkeyPatch) -> None: |
| urls: list[str] = [] |
|
|
| def handler(request: httpx.Request) -> httpx.Response: |
| urls.append(str(request.url)) |
| return httpx.Response(200, json={"message": {"content": "ok"}, "done": True}) |
|
|
| monkeypatch.setenv("MINDXTRAIN_BANKML_BASE_URL", "http://127.0.0.1:18111/v1") |
| probe = probe_recall_via_bankml("x", ["a"], transport=httpx.MockTransport(handler)) |
| assert urls == ["http://127.0.0.1:18111/api/chat"] |
| assert probe.receipts == [None] and probe.model_sha256 == [] |
|
|
