File size: 2,745 Bytes
1d2de8a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | import hashlib
import json
import numpy as np
import pytest
from solomon_mlx.artifacts import BASE_REVISION, verify_release
from solomon_mlx.evaluation import compare_rows, unit
from solomon_mlx.prepare import verify_base
def test_corrupt_source_and_base_fail_closed(tmp_path):
path = tmp_path / "input"
path.write_bytes(b"original")
sha = hashlib.sha256(b"original").hexdigest()
(tmp_path / "MANIFEST.json").write_text(
json.dumps({"files": [{"destination": "input", "staged_bytes": 8, "staged_sha256": sha}]})
)
assert verify_release(tmp_path)["input"] == sha
manifest = {"revision": BASE_REVISION, "files": [{"name": "input", "size": 8, "sha256": sha}]}
assert verify_base(tmp_path, manifest)["input"] == sha
path.write_bytes(b"corrupt!")
with pytest.raises(ValueError):
verify_release(tmp_path)
with pytest.raises(ValueError):
verify_base(tmp_path, manifest)
def test_reserved_choice_exclusion_and_whole_question_accuracy():
rows = [
{
"id": "a",
"task": "entity",
"n": 4,
"head_key": "entity/state4",
"question_id": "q",
"gold": 0,
"letter_logits": [5, 0, 0, 0],
},
{
"id": "b",
"task": "entity",
"n": 4,
"head_key": "entity/state4",
"question_id": "q",
"gold": 3,
"letter_logits": [0, 0, 0, 5],
},
]
other = [{**rows[0]}, {**rows[1], "letter_logits": [5, 0, 0, 0]}]
report = compare_rows(rows, other)
assert report["unit_decision_agreement"] == 0.5
assert report["mlx_whole_question_accuracy"] == 1
assert report["cuda_whole_question_accuracy"] == 0
assert not report["quality_gate_passed"]
assert unit(
{"task": "single", "n": 4, "head_key": "single/choiceR", "gold": 3, "letter_logits": [1, 2, 3, 4]}
) == (None, None)
def test_norm_offset_preserves_bf16_source_values_in_fp32():
from types import SimpleNamespace
import mlx.core as mx
from mlx_vlm.models.qwen3_5.qwen3_5 import Model
# Sanitization requires config only for tied embeddings and image channels.
config = SimpleNamespace(
text_config=SimpleNamespace(tie_word_embeddings=False), vision_config=SimpleNamespace(in_channels=3)
)
original = mx.array([0.02, -0.04], dtype=mx.bfloat16)
converted = Model.sanitize(
SimpleNamespace(config=config), {"model.language_model.norm.weight": original.astype(mx.float32)}
)
out = converted["language_model.model.norm.weight"]
assert out.dtype == mx.float32
np.testing.assert_array_equal(np.asarray(out), np.asarray(original.astype(mx.float32)) + 1)
|