SCQC / tests /test_text_align.py
raianand's picture
SCQC demo
912d2b5
Raw History Blame Contribute Delete
1.23 kB
from scqc.align import align, char_error_rate
from scqc.textnorm import NormConfig, normalize, tokenize
def test_punctuation_danda_and_digits_fold():
assert normalize("रेलगाड़ी प्लेटफॉर्म नंबर ३ पर आएगी।") == "रेलगाडी प्लेटफॉर्म नंबर 3 पर आएगी"
def test_nukta_folding_is_configurable():
assert normalize("दरवाज़ा") == normalize("दरवाजा")
assert normalize("दरवाज़ा", NormConfig(fold_nukta=False)) != normalize("दरवाजा", NormConfig(fold_nukta=False))
def test_precomposed_nukta_letter_folds():
assert normalize("\u095B") == "\u091C" # ज़ (precomposed) -> ज
def test_zero_width_removed():
assert normalize("क्\u200dष") == normalize("क्ष")
def test_alignment_ops():
al = align(["a", "b", "c", "d"], ["a", "x", "d", "e"])
ops = [o.op for o in al.ops]
assert ops[0] == "match" and al.errors == 3
assert al.error_rate == 0.75
def test_cer_ignores_spaces():
assert char_error_rate("आज मौसम", "आजमौसम") == 0.0
def test_tokenize_empty():
assert tokenize("। , !") == []