File size: 1,755 Bytes
78ac017 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | import json
from pathlib import Path
import pytest
from experiments.evaluate_qwen3vl_recent4_full import validate_processor_contract
from scripts.configure_evaluation_protocol import configure
from viss_core.evaluation_protocol import ProtocolValidationError, load_locked_protocol
TEMPLATE = Path(__file__).resolve().parents[1] / "configs/evaluation/recent4.example.json"
def test_unconfigured_template_cannot_be_used_as_a_locked_protocol():
with pytest.raises(ProtocolValidationError, match="not locked"):
load_locked_protocol(TEMPLATE)
def test_user_configured_protocol_preserves_processor_identity_check(tmp_path):
template = json.loads(TEMPLATE.read_text())
contract = template["inference"]["processor_contract"]
processor = tmp_path / "processor"
processor.mkdir()
config = processor / "preprocessor_config.json"
config.write_text(json.dumps({
key: contract[key]
for key in ["image_processor_type", "patch_size", "merge_size", "temporal_patch_size"]
} | {"size": {
key: contract[key] for key in ["shortest_edge", "longest_edge"]
}}))
streaming = tmp_path / "streaming.jsonl"
ovo = tmp_path / "ovo.jsonl"
for path in (streaming, ovo):
path.write_text('{"uid": "synthetic-query"}\n')
output = tmp_path / "configured.json"
configure(
template=TEMPLATE,
streaming_samples=streaming,
ovo_samples=ovo,
processor_config=config,
output=output,
)
protocol = load_locked_protocol(output)
validate_processor_contract(processor, protocol)
config.write_text(config.read_text() + " ")
with pytest.raises(ValueError, match="SHA-256 diverged"):
validate_processor_contract(processor, protocol)
|