ocr-projects-archive / ocr-tech /tests /test_audit_ctc_label_feasibility.py
prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
2.38 kB
from __future__ import annotations
import importlib.util
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location(
"audit_ctc_label_feasibility",
ROOT / "tools" / "audit_ctc_label_feasibility.py",
)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_ctc_min_timesteps_counts_adjacent_equal_tokens() -> None:
assert MODULE.ctc_min_timesteps("𑒏𑒏𑒫") == 4
assert MODULE.ctc_min_timesteps("𑒏𑒫𑒏") == 3
assert MODULE.ctc_min_timesteps("𑒏𑒏𑒏") == 5
def test_report_fails_impossible_label_and_passes_wider_pool(tmp_path: Path) -> None:
dictionary = tmp_path / "dict.txt"
dictionary.write_text("𑒏\nπ‘’«\n", encoding="utf-8")
labels = tmp_path / "labels.txt"
labels.write_text("a.png\t𑒏𑒏𑒫\nb.png\t𑒏𑒫\n", encoding="utf-8")
failed = MODULE.build_report(
dictionary=dictionary,
label_files=[("train", labels)],
pool_width=3,
max_text_length=2,
use_space_char=False,
)
assert failed["status"] == "FAIL"
assert failed["splits"][0]["over_pool_width"] == 1
assert failed["splits"][0]["over_max_text_length"] == 1
passed = MODULE.build_report(
dictionary=dictionary,
label_files=[("train", labels)],
pool_width=4,
max_text_length=3,
use_space_char=False,
)
assert passed["status"] == "PASS"
def test_multicodepoint_dictionary_is_rejected(tmp_path: Path) -> None:
dictionary = tmp_path / "dict.txt"
dictionary.write_text("𑒏𑓂\n", encoding="utf-8")
with pytest.raises(ValueError, match="exactly one codepoint"):
MODULE.audit_dictionary(dictionary, use_space_char=False)
def test_unknown_label_codepoint_is_a_failure(tmp_path: Path) -> None:
dictionary = tmp_path / "dict.txt"
dictionary.write_text("𑒏\n", encoding="utf-8")
labels = tmp_path / "labels.txt"
labels.write_text("a.png\t𑒏\u00a0𑒏\n", encoding="utf-8")
report = MODULE.build_report(
dictionary=dictionary,
label_files=[("train", labels)],
pool_width=10,
max_text_length=10,
use_space_char=True,
)
assert report["status"] == "FAIL"
assert report["splits"][0]["unknown_codepoints"] == ["U+00A0"]