from __future__ import annotations import importlib.util from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location( "audit_ctc_label_feasibility", ROOT / "tools" / "audit_ctc_label_feasibility.py", ) assert SPEC and SPEC.loader MODULE = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(MODULE) def test_ctc_min_timesteps_counts_adjacent_equal_tokens() -> None: assert MODULE.ctc_min_timesteps("š‘’š‘’š‘’«") == 4 assert MODULE.ctc_min_timesteps("š‘’š‘’«š‘’") == 3 assert MODULE.ctc_min_timesteps("š‘’š‘’š‘’") == 5 def test_report_fails_impossible_label_and_passes_wider_pool(tmp_path: Path) -> None: dictionary = tmp_path / "dict.txt" dictionary.write_text("š‘’\nš‘’«\n", encoding="utf-8") labels = tmp_path / "labels.txt" labels.write_text("a.png\tš‘’š‘’š‘’«\nb.png\tš‘’š‘’«\n", encoding="utf-8") failed = MODULE.build_report( dictionary=dictionary, label_files=[("train", labels)], pool_width=3, max_text_length=2, use_space_char=False, ) assert failed["status"] == "FAIL" assert failed["splits"][0]["over_pool_width"] == 1 assert failed["splits"][0]["over_max_text_length"] == 1 passed = MODULE.build_report( dictionary=dictionary, label_files=[("train", labels)], pool_width=4, max_text_length=3, use_space_char=False, ) assert passed["status"] == "PASS" def test_multicodepoint_dictionary_is_rejected(tmp_path: Path) -> None: dictionary = tmp_path / "dict.txt" dictionary.write_text("š‘’š‘“‚\n", encoding="utf-8") with pytest.raises(ValueError, match="exactly one codepoint"): MODULE.audit_dictionary(dictionary, use_space_char=False) def test_unknown_label_codepoint_is_a_failure(tmp_path: Path) -> None: dictionary = tmp_path / "dict.txt" dictionary.write_text("š‘’\n", encoding="utf-8") labels = tmp_path / "labels.txt" labels.write_text("a.png\tš‘’\u00a0š‘’\n", encoding="utf-8") report = MODULE.build_report( dictionary=dictionary, label_files=[("train", labels)], pool_width=10, max_text_length=10, use_space_char=True, ) assert report["status"] == "FAIL" assert report["splits"][0]["unknown_codepoints"] == ["U+00A0"]