| import hashlib |
| import json |
| from pathlib import Path |
|
|
| import pytest |
|
|
| from br_sovereign_llm.tokenization.tokenizer_export import ( |
| TokenizerExportError, |
| validate_tokenizer_export, |
| ) |
|
|
|
|
| def test_tokenizer_export_digest_is_verified(tmp_path: Path) -> None: |
| tokenizer = tmp_path / "tokenizer.json" |
| tokenizer.write_text('{"version":"fixture"}\n', encoding="utf-8") |
| digest = hashlib.sha256(tokenizer.read_bytes()).hexdigest() |
| manifest = { |
| "tokenizer_sha256": digest, |
| "tokenizer_id": "fixture", |
| } |
| (tmp_path / "tokenizer-manifest.json").write_text( |
| json.dumps(manifest), encoding="utf-8" |
| ) |
|
|
| observed = validate_tokenizer_export(tmp_path) |
|
|
| assert observed["tokenizer_id"] == "fixture" |
|
|
|
|
| def test_tokenizer_export_rejects_modified_artifact(tmp_path: Path) -> None: |
| tokenizer = tmp_path / "tokenizer.json" |
| tokenizer.write_text('{"version":"before"}\n', encoding="utf-8") |
| manifest = { |
| "tokenizer_sha256": hashlib.sha256(tokenizer.read_bytes()).hexdigest(), |
| } |
| (tmp_path / "tokenizer-manifest.json").write_text( |
| json.dumps(manifest), encoding="utf-8" |
| ) |
| tokenizer.write_text('{"version":"after"}\n', encoding="utf-8") |
|
|
| with pytest.raises(TokenizerExportError, match="digest mismatch"): |
| validate_tokenizer_export(tmp_path) |
|
|
