from scqc.align import align, char_error_rate from scqc.textnorm import NormConfig, normalize, tokenize def test_punctuation_danda_and_digits_fold(): assert normalize("रेलगाड़ी प्लेटफॉर्म नंबर ३ पर आएगी।") == "रेलगाडी प्लेटफॉर्म नंबर 3 पर आएगी" def test_nukta_folding_is_configurable(): assert normalize("दरवाज़ा") == normalize("दरवाजा") assert normalize("दरवाज़ा", NormConfig(fold_nukta=False)) != normalize("दरवाजा", NormConfig(fold_nukta=False)) def test_precomposed_nukta_letter_folds(): assert normalize("\u095B") == "\u091C" # ज़ (precomposed) -> ज def test_zero_width_removed(): assert normalize("क्\u200dष") == normalize("क्ष") def test_alignment_ops(): al = align(["a", "b", "c", "d"], ["a", "x", "d", "e"]) ops = [o.op for o in al.ops] assert ops[0] == "match" and al.errors == 3 assert al.error_rate == 0.75 def test_cer_ignores_spaces(): assert char_error_rate("आज मौसम", "आजमौसम") == 0.0 def test_tokenize_empty(): assert tokenize("। , !") == []