Download tests/test_text_align.py from raianand/SCQC: direct link, hf CLI and curl.
- Browser
- Download file 1.23 kB
-
https://huggingface.co/spaces/raianand/SCQC/resolve/main/tests/test_text_align.py
- Command line
-
hf download hf://spaces/raianand/SCQC/tests/test_text_align.py
-
curl -L -o test_text_align.py https://huggingface.co/spaces/raianand/SCQC/resolve/main/tests/test_text_align.py
1.23 kB
| from scqc.align import align, char_error_rate | |
| from scqc.textnorm import NormConfig, normalize, tokenize | |
| def test_punctuation_danda_and_digits_fold(): | |
| assert normalize("रेलगाड़ी प्लेटफॉर्म नंबर ३ पर आएगी।") == "रेलगाडी प्लेटफॉर्म नंबर 3 पर आएगी" | |
| def test_nukta_folding_is_configurable(): | |
| assert normalize("दरवाज़ा") == normalize("दरवाजा") | |
| assert normalize("दरवाज़ा", NormConfig(fold_nukta=False)) != normalize("दरवाजा", NormConfig(fold_nukta=False)) | |
| def test_precomposed_nukta_letter_folds(): | |
| assert normalize("\u095B") == "\u091C" # ज़ (precomposed) -> ज | |
| def test_zero_width_removed(): | |
| assert normalize("क्\u200dष") == normalize("क्ष") | |
| def test_alignment_ops(): | |
| al = align(["a", "b", "c", "d"], ["a", "x", "d", "e"]) | |
| ops = [o.op for o in al.ops] | |
| assert ops[0] == "match" and al.errors == 3 | |
| assert al.error_rate == 0.75 | |
| def test_cer_ignores_spaces(): | |
| assert char_error_rate("आज मौसम", "आजमौसम") == 0.0 | |
| def test_tokenize_empty(): | |
| assert tokenize("। , !") == [] | |