import hashlib import json from pathlib import Path import pytest from br_sovereign_llm.tokenization.tokenizer_export import ( TokenizerExportError, validate_tokenizer_export, ) def test_tokenizer_export_digest_is_verified(tmp_path: Path) -> None: tokenizer = tmp_path / "tokenizer.json" tokenizer.write_text('{"version":"fixture"}\n', encoding="utf-8") digest = hashlib.sha256(tokenizer.read_bytes()).hexdigest() manifest = { "tokenizer_sha256": digest, "tokenizer_id": "fixture", } (tmp_path / "tokenizer-manifest.json").write_text( json.dumps(manifest), encoding="utf-8" ) observed = validate_tokenizer_export(tmp_path) assert observed["tokenizer_id"] == "fixture" def test_tokenizer_export_rejects_modified_artifact(tmp_path: Path) -> None: tokenizer = tmp_path / "tokenizer.json" tokenizer.write_text('{"version":"before"}\n', encoding="utf-8") manifest = { "tokenizer_sha256": hashlib.sha256(tokenizer.read_bytes()).hexdigest(), } (tmp_path / "tokenizer-manifest.json").write_text( json.dumps(manifest), encoding="utf-8" ) tokenizer.write_text('{"version":"after"}\n', encoding="utf-8") with pytest.raises(TokenizerExportError, match="digest mismatch"): validate_tokenizer_export(tmp_path)