| import collections |
| import itertools |
| import os |
| import re |
|
|
| try: |
| import jamo |
| except ImportError: |
| jamo = None |
|
|
| WIKIPRON_DIR = r"C:\Users\micha\Documents\Dev\wikipron\data\scrape\tsv" |
| OUTPUT_DIR = r"" |
|
|
| LANG_PATHS = {"french": "fra_latn_broad_filtered.tsv"} |
|
|
| |
| |
| |
| |
| |
| |
| LANG_CODES = ["czech"] |
|
|
| BAD_GRAPHEMES = { |
| "english_us": { |
| "%", |
| "/", |
| "@", |
| "²", |
| "à", |
| "á", |
| "â", |
| "ä", |
| "æ", |
| "ç", |
| "è", |
| "é", |
| "ê", |
| "ë", |
| "í", |
| "î", |
| "ï", |
| "ñ", |
| "ó", |
| "ô", |
| "õ", |
| "ö", |
| "ø", |
| "ù", |
| "ú", |
| "ü", |
| "ā", |
| "ą", |
| "č", |
| "ē", |
| "ę", |
| "ğ", |
| "ı", |
| "ł", |
| "ń", |
| "ō", |
| "ő", |
| "œ", |
| "ř", |
| "ū", |
| "ș", |
| "ț", |
| "ʼ", |
| "ṭ", |
| "₂", |
| }, |
| "english_uk": { |
| "%", |
| "/", |
| "@", |
| "²", |
| "à", |
| "á", |
| "â", |
| "ä", |
| "æ", |
| "ç", |
| "è", |
| "é", |
| "ê", |
| "ë", |
| "í", |
| "î", |
| "ï", |
| "ñ", |
| "ó", |
| "ô", |
| "õ", |
| "ö", |
| "ø", |
| "ù", |
| "ú", |
| "ü", |
| "ā", |
| "ą", |
| "č", |
| "ē", |
| "ę", |
| "ğ", |
| "ı", |
| "ł", |
| "ń", |
| "ō", |
| "ő", |
| "œ", |
| "ř", |
| "ū", |
| "ș", |
| "ț", |
| "ʼ", |
| "ṭ", |
| "₂", |
| "ã", |
| "å", |
| "û", |
| "ī", |
| "ž", |
| ".", |
| }, |
| "polish": {"+", ".", "ü", "ö", "ø", "ƶ", "ñ", "ç", "à", "á", "è", "é", "í"}, |
| "french": {".", "/", "º", "å", "æ", "ÿ", "ș"}, |
| "japanese": {" ", "&", "+", "、", "〆", "〼", "〼", "＝", "𫡤", "・", "×", "ゞ", "ゟ", "ゑ", "ゐ", "ヲ"}, |
| "mandarin_hani_beijing": { |
| "A", |
| "B", |
| "C", |
| "D", |
| "E", |
| "G", |
| "H", |
| "I", |
| "K", |
| "M", |
| "N", |
| "O", |
| "P", |
| "Q", |
| "S", |
| "T", |
| "U", |
| "V", |
| "X", |
| "Y", |
| "Z", |
| "e", |
| "p", |
| "u", |
| "·", |
| "α", |
| "β", |
| "γ", |
| "…", |
| "⿰", |
| "ㄅ", |
| "ㄆ", |
| "ㄇ", |
| "ㄈ", |
| "𰚼", |
| "𰯼", |
| "𫇦", |
| }, |
| "mandarin_hani_taiwan": { |
| "A", |
| "B", |
| "C", |
| "D", |
| "E", |
| "G", |
| "H", |
| "I", |
| "K", |
| "M", |
| "N", |
| "O", |
| "P", |
| "Q", |
| "S", |
| "T", |
| "U", |
| "V", |
| "X", |
| "Y", |
| "Z", |
| "e", |
| "p", |
| "u", |
| "·", |
| "α", |
| "β", |
| "γ", |
| "…", |
| "⿰", |
| "ㄅ", |
| "ㄆ", |
| "ㄇ", |
| "ㄈ", |
| "𰚼", |
| "𰯼", |
| "𫇦", |
| }, |
| "mandarin_hani_standard": { |
| "A", |
| "B", |
| "C", |
| "D", |
| "E", |
| "G", |
| "H", |
| "I", |
| "K", |
| "M", |
| "N", |
| "O", |
| "P", |
| "Q", |
| "S", |
| "T", |
| "U", |
| "V", |
| "X", |
| "Y", |
| "Z", |
| "e", |
| "p", |
| "u", |
| "·", |
| "α", |
| "β", |
| "γ", |
| "…", |
| "⿰", |
| "ㄅ", |
| "ㄆ", |
| "ㄇ", |
| "ㄈ", |
| "𰚼", |
| "𰯼", |
| "𫇦", |
| }, |
| "german": {"'", ".", "@", "à", "á", "ç", "è", "é", "ê", "ó", "ø", "œ", "í", "ë"}, |
| "portuguese_brazil": {"'", "."}, |
| "portuguese_portugal": {"'", "."}, |
| "russian": {"'", ".", "/", "ѳ"}, |
| "spanish_spain": {"'", ".", "ö", "ꝇ", "î", "ç"}, |
| "spanish_latin_america": {"'", ".", "ö", "ꝇ", "î", "ç"}, |
| "thai": {"…", "'", "/"}, |
| "turkish": {"̇", "'"}, |
| "tamil": {"ࢳ", "ࢳ", "ࢴ", "ࢴ", "ஃ"}, |
| "vietnamese_hanoi": { |
| "'", |
| ".", |
| ",", |
| }, |
| "vietnamese_hue": { |
| "'", |
| ".", |
| ",", |
| }, |
| "vietnamese_hochiminhcity": { |
| "'", |
| ".", |
| ",", |
| }, |
| } |
|
|
|
|
| BAD_PHONES = { |
| "english_uk": {"ɪː", "aː", "eː", "a", "o", "oː", "eː", "e"}, |
| "english_us": {"ɒ", "aː", "a", "o", "oː", "eː", "e", "ɪː", "ɛː"}, |
| "german": {"ʊɪ"}, |
| "czech": {"ə"}, |
| "spanish_latin_america": {"ɹ", "ɚ", "ʒ", "ə", "ɪ"}, |
| "spanish_spain": {"ɹ", "ɚ", "ʒ", "ə", "ɪ"}, |
| "mandarin_hani_taiwan": {"ai", "a", "ei", "o", "ə", "z̩", "ʐ̩"}, |
| "mandarin_hani_standard": {"ai", "a", "ei", "o", "ə", "z̩", "ʐ̩"}, |
| "mandarin_hani_beijing": {"ai", "a", "ei", "o", "ə", "z̩", "ʐ̩"}, |
| } |
|
|
| VOWELS = { |
| "english_us": { |
| "aɪ", |
| "aʊ", |
| "eɪ", |
| "i", |
| "iː", |
| "oɪ", |
| "oʊ", |
| "u", |
| "uː", |
| "æ", |
| "ɑ", |
| "ɑː", |
| "ɔ", |
| "ɔɪ", |
| "ɔː", |
| "ə", |
| "ɚ", |
| "ɛ", |
| "ɝ", |
| "ɝː", |
| "ɪ", |
| "ʊ", |
| "ʌ", |
| }, |
| "english_uk": { |
| "aɪ", |
| "aʊ", |
| "eɪ", |
| "i", |
| "iː", |
| "oɪ", |
| "oʊ", |
| "u", |
| "uː", |
| "æ", |
| "ɑ", |
| "ɑː", |
| "ɔ", |
| "ɔɪ", |
| "ɔː", |
| "ə", |
| "ɚ", |
| "ɛ", |
| "ɝ", |
| "ɝː", |
| "ɪ", |
| "ʊ", |
| "ʌ", |
| "aɪ", |
| "aʊ", |
| "eɪ", |
| "i", |
| "iː", |
| "oɪ", |
| "oʊ", |
| "u", |
| "uː", |
| "æ", |
| "ɑ", |
| "ɑː", |
| "ɒ", |
| "ɔ", |
| "ɔɪ", |
| "ɔː", |
| "ɛ", |
| "ɛː", |
| "ɜ", |
| "ɜː", |
| "ʊ", |
| "ʌ", |
| }, |
| "vietnamese_hanoi": {"a", "aː", "e", "i", "o", "u", "ɔ", "ə", "əː", "ɛ", "ɨ", "ʊ", "ɪ"}, |
| "vietnamese_hue": {"a", "aː", "e", "i", "o", "u", "ɔ", "ə", "əː", "ɛ", "ɨ", "ʊ", "ɪ"}, |
| "vietnamese_hochiminhcity": { |
| "a", |
| "aː", |
| "e", |
| "i", |
| "o", |
| "u", |
| "ɔ", |
| "ə", |
| "əː", |
| "ɛ", |
| "ɨ", |
| "ʊ", |
| "ɪ", |
| }, |
| "mandarin_hani": {"a", "e", "o", "i", "u", "y", "ə", "ɚ", "ɤ̃", "ʊ̃"}, |
| "mandarin_hani_standard": {"a", "e", "o", "i", "u", "y", "ə", "ɚ", "ɤ̃", "ʊ̃"}, |
| "mandarin_hani_taiwan": {"a", "e", "o", "i", "u", "y", "ə", "ɚ", "ɤ̃", "ʊ̃"}, |
| "mandarin_hani_beijing": {"a", "e", "o", "i", "u", "y", "ə", "ɚ", "ɤ̃", "ʊ̃"}, |
| "thai": { |
| "a", |
| "aː", |
| "e", |
| "eː", |
| "i", |
| "iː", |
| "o", |
| "oː", |
| "u", |
| "ə", |
| "uː", |
| "ɔ", |
| "ɔː", |
| "ɛ", |
| "ɛː", |
| "ɤ", |
| "ɤː", |
| "ɯ", |
| "ɯː", |
| }, |
| "swedish": { |
| "a", |
| "aʊ", |
| "aː", |
| "e", |
| "eː", |
| "i", |
| "iː", |
| "o", |
| "oː", |
| "u", |
| "uː", |
| "y", |
| "yʷ", |
| "yː", |
| "æ", |
| "æː", |
| "êː", |
| "ø", |
| "øː", |
| "ø̀ː", |
| "œ", |
| "œː", |
| "œ̞", |
| "œ̞ː", |
| "ɑ", |
| "ɑː", |
| "ɒː", |
| "ɒ̀ː", |
| "ɔ", |
| "ə", |
| "ɚ", |
| "ɛ", |
| "ɛɵ", |
| "ɛː", |
| "ɛ̂", |
| "ɛ̄", |
| "ɜ", |
| "ɝ", |
| "ɪ", |
| "ɵ", |
| "ɵː", |
| "ɵ̄", |
| "ɶ", |
| "ɶː", |
| "ʉ", |
| "ʉː", |
| "ʉ̂ː", |
| "ʉ̟ː", |
| "ʊ", |
| "ʊː", |
| "ʏ", |
| "ỳː", |
| "ỵː", |
| }, |
| } |
|
|
| VOWEL_PATTERNS = {"swedish": re.compile(r"^[aeiɛøæuoʊêɔɪœɑʉɵɶ̂œ̞ː˧˩ɒyʏʉ̟ː˧˩əː˧˩˥]+$")} |
|
|
|
|
| LANG_MAPPING = { |
| "bulgarian": { |
| "d̪": "d", |
| "t̪": "t", |
| "ɐ": "a", |
| "æ": "a", |
| "a̟": "a", |
| "e": "ɛ", |
| "ə": "ɤ", |
| "o̝": "ɔ", |
| "o̟": "ɔ", |
| "u̟": "u", |
| "ʉ": "u", |
| "ʊ": "u", |
| "ɤ̞": "ɤ", |
| "ɤ̟": "ɤ", |
| "lʲ": "ʎ", |
| "l": "ɫ", |
| "ɾ": "r", |
| "iː": "i j", |
| "s̪": "s", |
| "n̪": "n", |
| "ɾʲ": "rʲ", |
| "nʲ": "ɲ", |
| "ɡʲ": "ɟ", |
| "kʲ": "c", |
| }, |
| "russian": {}, |
| "czech": { |
| "ɫ": "l", |
| "ɾ": "r", |
| "ɔ": "o", |
| "ɔː": "oː", |
| }, |
| "serbocroatian_croatian": { |
| "ʋ": "v", |
| "ɕ": "ʃ", |
| "ʑ": "ʒ", |
| "ô": "o˦˨", |
| "ôː": "oː˦˨", |
| "ûː": "uː˦˨", |
| "û": "u˦˨", |
| "î": "i˦˨", |
| "îː": "iː˦˨", |
| "êː": "eː˦˨", |
| "ê": "e˦˨", |
| "âː": "aː˦˨", |
| "â": "a˦˨", |
| "r̂": "r̩˦˨", |
| "r̂ː": "r̩ː˦˨", |
| "řː": "r̩ː˨˦", |
| "ř": "r̩˨˦", |
| "ěː": "eː˨˦", |
| "ě": "e˨˦", |
| "ǎ": "a˨˦", |
| "ǎː": "aː˨˦", |
| "ǐː": "iː˨˦", |
| "ǐ": "i˨˦", |
| "ǒ": "o˨˦", |
| "ǒː": "oː˨˦", |
| "ǔː": "uː˨˦", |
| "ǔ": "u˨˦", |
| }, |
| "serbocroatian_serbian": { |
| "ʋ": "v", |
| "ɕ": "ʃ", |
| "ʑ": "ʒ", |
| "ô": "o˦˨", |
| "ôː": "oː˦˨", |
| "ûː": "uː˦˨", |
| "û": "u˦˨", |
| "î": "i˦˨", |
| "îː": "iː˦˨", |
| "êː": "eː˦˨", |
| "ê": "e˦˨", |
| "âː": "aː˦˨", |
| "â": "a˦˨", |
| "r̂": "r̩˦˨", |
| "r̂ː": "r̩ː˦˨", |
| "řː": "r̩ː˨˦", |
| "ř": "r̩˨˦", |
| "ěː": "eː˨˦", |
| "ě": "e˨˦", |
| "ǎ": "a˨˦", |
| "ǎː": "aː˨˦", |
| "ǐː": "iː˨˦", |
| "ǐ": "i˨˦", |
| "ǒ": "o˨˦", |
| "ǒː": "oː˨˦", |
| "ǔː": "uː˨˦", |
| "ǔ": "u˨˦", |
| }, |
| "french": {"r": "ʁ", "œ̃": "ɛ̃"}, |
| "vietnamese_hanoi": { |
| "k̟̚": "k̚", |
| "ŋ̟": "ŋ", |
| "ï": "ɨ", |
| }, |
| "german": { |
| "b̥": "b", |
| "d̥": "d", |
| "ɡ̊": "ɡ", |
| "r": "ʁ", |
| "ŋ̍": "n̩", |
| "ɱ̩": "n̩", |
| "ŋ̩": "n̩", |
| "ʀ": "ʁ", |
| "χ": "x", |
| "ʋ": "v", |
| "ɘ": "ə", |
| "i": "ɪ", |
| "ø": "øː", |
| "o": "ɔ", |
| "u": "ʊ", |
| "œː": "øː", |
| "y": "ʏ", |
| "e": "ɛ", |
| "ɛː": "eː", |
| "ɔː": "oː", |
| "ɑː": "aː", |
| "ɒː": "aː", |
| }, |
| "mandarin_hani": { |
| "b̥": "p", |
| "d̥": "t", |
| "g̊": "k", |
| "ɡ̊": "k", |
| "ɖʐ̥": "ʈʂ", |
| "dz̥": "ts", |
| "dʑ̥": "tɕ", |
| "ä": "a", |
| "æ̃": "a", |
| "ɤ": "o", |
| "ɤ̃": "o", |
| "ʊ̃": "o", |
| "ɪ": "i", |
| "ɻʷ": "ɻ", |
| "ʊ": "u", |
| "ɛ": "e", |
| "ɑ": "a", |
| "ɑ̃": "a", |
| "ɔ": "o", |
| "ɔː": "o", |
| "⁵⁵": "˥", |
| "⁵¹": "˥˩", |
| "³⁵": "˧˥", |
| "²¹⁴": "˨˩˦", |
| }, |
| "mandarin_hani_taiwan": { |
| "b̥": "p", |
| "d̥": "t", |
| "g̊": "k", |
| "ɡ̊": "k", |
| "ɖʐ̥": "ʈʂ", |
| "dz̥": "ts", |
| "dʑ̥": "tɕ", |
| "ä": "a", |
| "æ̃": "a", |
| "ɤ": "o", |
| "ɤ̃": "o", |
| "ʊ̃": "o", |
| "ɪ": "i", |
| "ɻʷ": "ɻ", |
| "ʊ": "u", |
| "ɛ": "e", |
| "ɑ": "a", |
| "ɑ̃": "a", |
| "ɔ": "o", |
| "ɔː": "o", |
| "⁵⁵": "˥", |
| "⁵¹": "˥˩", |
| "³⁵": "˧˥", |
| "²¹⁴": "˨˩˦", |
| }, |
| "mandarin_hani_beijing": { |
| "b̥": "p", |
| "d̥": "t", |
| "g̊": "k", |
| "ɡ̊": "k", |
| "ɖʐ̥": "ʈʂ", |
| "dz̥": "ts", |
| "dʑ̥": "tɕ", |
| "ä": "a", |
| "æ̃": "a", |
| "ɤ": "o", |
| "ɤ̃": "o", |
| "ʊ̃": "o", |
| "ɪ": "i", |
| "ɻʷ": "ɻ", |
| "ʊ": "u", |
| "ɛ": "e", |
| "ɑ": "a", |
| "ɑ̃": "a", |
| "ɔ": "o", |
| "ɔː": "o", |
| "⁵⁵": "˥", |
| "⁵¹": "˥˩", |
| "³⁵": "˧˥", |
| "²¹⁴": "˨˩˦", |
| }, |
| "mandarin_hani_standard": { |
| "b̥": "p", |
| "d̥": "t", |
| "g̊": "k", |
| "ɡ̊": "k", |
| "ɖʐ̥": "ʈʂ", |
| "dz̥": "ts", |
| "dʑ̥": "tɕ", |
| "ä": "a", |
| "æ̃": "a", |
| "ɤ": "o", |
| "ɤ̃": "o", |
| "ʊ̃": "o", |
| "ɪ": "i", |
| "ɻʷ": "ɻ", |
| "ʊ": "u", |
| "ɛ": "e", |
| "ɑ": "a", |
| "ɑ̃": "a", |
| "ɔ": "o", |
| "ɔː": "o", |
| "⁵⁵": "˥", |
| "⁵¹": "˥˩", |
| "³⁵": "˧˥", |
| "²¹⁴": "˨˩˦", |
| }, |
| "polish": { |
| "s̪": "s", |
| "r̥ː": "r", |
| "r̥": "r", |
| "ɫ": "l", |
| "w̃": "n", |
| }, |
| "portuguese_brazil": { |
| "ã": "ɐ̃", |
| "ɫ": "l", |
| "ʁ": "x", |
| "ɹ": "x", |
| "ɻ": "x", |
| "χ": "x", |
| "ɦ": "x", |
| "h": "x", |
| "r": "x", |
| "ɪ": "i", |
| "ʊ": "u", |
| }, |
| "portuguese_portugal": { |
| "ã": "ɐ̃", |
| "ɫ": "l", |
| "r": "ʁ", |
| }, |
| "swedish": { |
| "ɛ̄": "ɛ̂", |
| "ɵ̄": "ɵ̂", |
| "ɘ": "ɵ", |
| "ə": "ɛ", |
| "ʁ": "r", |
| "ɾ": "r", |
| "ɹ": "r", |
| "v": "ʋ", |
| "w": "ʋ", |
| "ɜ": "ɛ", |
| "æː": "ɛː", |
| "ø": "øː", |
| "æ": "ɛ", |
| "ˇl": "l", |
| "yʷ": "y", |
| "œ̞ː": "øː", |
| "œː": "øː", |
| "œ̞": "œ", |
| "ç": "ɕ", |
| "bː": "b", |
| "ɖː": "ɖ", |
| "ɖˑ": "ɖ", |
| "ˈt": "tʰ", |
| "ˈk": "kʰ", |
| "ˈp": "pʰ", |
| "dː": "d", |
| "jː": "j", |
| "kː": "kʰ", |
| "lː": "l", |
| "mː": "m", |
| "nː": "n", |
| "fː": "f", |
| "ɧː": "ɧ", |
| "pː": "pʰ", |
| "rː": "r", |
| "sː": "s", |
| "tˑ": "t", |
| "tʰː": "tʰ", |
| "pʰː": "pʰ", |
| "kʰː": "kʰ", |
| "tː": "tʰ", |
| "ŋː": "ŋ", |
| "ɲ": "ɳ", |
| "ɕː": "ɕ", |
| "ɡː": "ɡ", |
| "ʈː": "ʈʰ", |
| "ʈʰː": "ʈʰ", |
| "ʂː": "ʈ", |
| "ỵː": "yː", |
| "ʉ̟̂": "ʉ̂", |
| "ʉ̟ː": "ʉː", |
| "ʉ̂": "ʉ̂ː", |
| "ɒː": "ɑː", |
| "aː": "ɑː", |
| "ɑ": "ɑː", |
| "e": "eː", |
| "o": "oː", |
| "u": "uː", |
| "i": "iː", |
| "y": "yː", |
| "ɒ̀ː": "ɑ̀ː", |
| "ʊː": "ʊ", |
| "ʉ": "ʉː", |
| "ɵː": "uː", |
| "ɶː": "øː", |
| }, |
| "tamil": { |
| "l̪": "l", |
| "l̪ː": "lː", |
| "r̥": "r", |
| "ɾ̪": "ɾ", |
| "h": "ɦ", |
| "tʃ": "tɕ", |
| "ɕ": "tɕ", |
| "tʃː": "tɕː", |
| }, |
| "thai": { |
| "cʰ": "tɕʰ", |
| "c": "tɕ", |
| "ɔ̌": "ɔ˩˩˦", |
| "ǎː": "aː˩˩˦", |
| "áː": "aː˦˥", |
| "à": "a˨˩", |
| "ì": "i˨˩", |
| }, |
| "ukrainian": { |
| "ɫ": "l", |
| "ʍ": "ʋ", |
| "w": "ʋ", |
| "v": "ʋ", |
| |
| |
| "ɫː": "lː", |
| }, |
| "japanese": { |
| "o̞": "o", |
| "n̩": "n", |
| "ä": "a", |
| "ɡ̊": "ɡ", |
| "ḁ": "a", |
| "ẽ": "e", |
| "m̩ː": "mː", |
| "e̥": "e", |
| "u͍": "ɯ", |
| "ɯ̃ᵝ": "ɯ", |
| "u͍ː": "ɯː", |
| "w͍": "w", |
| "y": "j", |
| "r": "ɾ", |
| "ɽ": "ɾ", |
| "ɾ̥": "ɾ", |
| "ɯᵝ": "ɯ", |
| "ɯᵝː": "ɯː", |
| "ɯ̟̃ᵝː": "ɯː", |
| "ɯ̥ᵝ": "ɯ̥", |
| "ʲkʲ": "kʲ", |
| "nʲ": "ɲ", |
| "tɕʲ": "tɕ", |
| "ɕʲ": "ɕ", |
| "ĩː": "iː", |
| "õ̞ː": "oː", |
| "i̥̥": "i̥", |
| "e̞̊": "e", |
| "ẽ̞ː": "eː", |
| "ã̠ː": "aː", |
| "õ̞": "o", |
| "d̥": "d", |
| "b̥": "b", |
| "o̞ː": "oː", |
| "e̞ː": "eː", |
| "e̞": "e", |
| "ẽ̞": "e", |
| "ĩ": "i", |
| "ɸ̥": "ɸ", |
| "ɨ̃ᵝː": "ɨː", |
| "ĩ̥": "i", |
| "a̠ː": "aː", |
| "a̠": "a", |
| "o̞̊": "o", |
| "dʑʲ": "dʑ", |
| "ɾ̠": "ɾ", |
| "ã̠": "a", |
| "õ̥": "o", |
| "dʲ": "dʑ", |
| "tʲ": "tɕ", |
| |
| "ɰᵝ": "w", |
| "ɰᵝː": "wː", |
| |
| |
| |
| |
| |
| |
| |
| "ɯ̟̊": "ɯ̥", |
| "ɲ̟": "ɲ", |
| "ŋʲ": "ɲ", |
| "p̚ʲ": "p̚", |
| "k̚ʲ": "k̚", |
| "t̚ʲ": "t̚", |
| }, |
| "turkish": { |
| "ɑ": "a", |
| "ɑː": "a", |
| "aː": "a", |
| "iː": "i", |
| "uː": "u", |
| "ɛ": "e", |
| "e̞": "e", |
| "ɔ": "o", |
| "ʊ": "u", |
| "ʏ": "y", |
| "β": "v", |
| "o̞": "o", |
| "ɪ": "i", |
| "ø": "œ", |
| "ɾ̝̊": "ɾ", |
| }, |
| "korean_hangul": { |
| "a̠": "a", |
| "e̞": "e", |
| "e̞ː": "eː", |
| "a̠ː": "a", |
| "o̞": "o", |
| "o̞ː": "oː", |
| "ʌ̹": "ʌ", |
| "ɘː": "ʌː", |
| "ɦ": "h", |
| "ɸʷ": "ɸ", |
| "ʃʰ": "sʰ", |
| }, |
| "english_uk": { |
| "ɝː": "ɜː", |
| "əː": "ɜː", |
| "æː": "æ", |
| "ɝ": "ɜ", |
| "ɚ": "ə", |
| "ɫ": "l", |
| "r": "ɹ", |
| "ʍ": "w", |
| }, |
| "english_us": { |
| "ɫ": "l", |
| "r": "ɹ", |
| "ʍ": "w", |
| "æː": "æ", |
| }, |
| "spanish_spain": { |
| "ɣ̞": "ɣ", |
| "β̞": "β", |
| "ð̞": "ð", |
| "θ̬": "θ", |
| "w̝": "w", |
| "nʲ": "ɲ", |
| "n̟": "n", |
| "lʲ": "ʎ", |
| "l̟": "l", |
| "i̯": "j", |
| "u̯": "w", |
| "h": "x", |
| "n̪": "n", |
| "d": "d̪", |
| }, |
| "spanish_latin_america": { |
| "ɣ̞": "ɣ", |
| "β̞": "β", |
| "ð̞": "ð", |
| "w̝": "w", |
| "nʲ": "ɲ", |
| "lʲ": "ʎ", |
| "i̯": "j", |
| "u̯": "w", |
| "n̪": "n", |
| "l̪": "l", |
| "l̟": "l", |
| "h": "x", |
| "n̟": "n", |
| "d": "d̪", |
| }, |
| } |
|
|
| GLOBAL_REMAPPING = { |
| "õ": "õ", |
| "ẽ": "ẽ", |
| "ũ": "ũ", |
| "ĩ": "ĩ", |
| "ã": "ã", |
| } |
|
|
|
|
| def read_source(lang): |
| graphemes = set() |
| phones = set() |
| dictionary = [] |
| path = os.path.join(WIKIPRON_DIR, LANG_PATHS[lang]) |
| with open(path, "r", encoding="utf8") as f: |
| for line in f: |
| line = line.strip() |
| if not line: |
| continue |
| if "\t" in line: |
| line = line.split("\t") |
| word = line[0] |
| pronunciation = line[1].split() |
| else: |
| line = line.split() |
| word = line[0] |
| pronunciation = line[1:] |
| word = word.lower() |
| if lang in BAD_GRAPHEMES: |
| if any(x in BAD_GRAPHEMES[lang] for x in word): |
| print(word) |
| continue |
| graphemes.update(word) |
| phones.update(pronunciation) |
| dictionary.append((word, pronunciation)) |
| return dictionary, graphemes, phones |
|
|
|
|
| def save_dictionary(dictionary, lang): |
| deduplication = set() |
| final_phones = collections.Counter() |
| path = os.path.join(OUTPUT_DIR, f"{lang}_mfa.dict") |
| with open(path, "w", encoding="utf8") as f: |
| for w, p in sorted(dictionary): |
| final_phones.update(p) |
| p = " ".join(p) |
| if (w, p) in deduplication: |
| continue |
| f.write("{}\t{}\n".format(w, p)) |
| deduplication.add((w, p)) |
| print("Final phones:", sorted(final_phones)) |
| print("Final phone counts:", sorted(final_phones.items(), key=lambda x: -x[1])) |
|
|
|
|
| def convert_language_specific(word, phones, lang): |
| new_pron = [] |
| if lang == "swedish": |
| for i, p in enumerate(phones): |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| phones[i] = v |
| break |
|
|
| for i, p in enumerate(phones): |
| if p == "¹": |
| found_first = False |
| found_second = False |
| for j in range(i + 1, len(phones)): |
| if VOWEL_PATTERNS[lang].match(phones[j]): |
| if not found_first: |
| phones[j] += "˥˧" |
| found_first = True |
| elif not found_second: |
| phones[j] += "˩" |
| found_second = True |
| else: |
| break |
| continue |
| elif p == "²": |
| found_first = False |
| found_second = False |
| for j in range(i + 1, len(phones)): |
| if phones[j] in VOWELS[lang]: |
| if not found_first: |
| phones[j] += "˧˩" |
| found_first = True |
| elif not found_second: |
| phones[j] += "˥˩" |
| found_second = True |
| else: |
| break |
| continue |
| new_pron.append(p) |
| phones = new_pron |
| new_pron = [] |
| for i, p in enumerate(phones): |
| if lang == "english_us": |
| if lang in LANG_MAPPING: |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
|
|
| if p == "ʒ" and len(new_pron) and new_pron[-1] == "d": |
| new_pron[-1] = "dʒ" |
| continue |
| elif ( |
| p == "ʃ" and len(new_pron) and new_pron[-1] == "t" |
| ): |
| new_pron[-1] = "tʃ" |
| continue |
| elif p in ["ɪ", "j"] and len(new_pron) and new_pron[-1] in {"e", "ɔ", "o"}: |
| new_pron[-1] += "ɪ" |
| continue |
| elif p in {"ʊ", "u"} and len(new_pron) and new_pron[-1] in {"a", "ɑ", "ʌ"}: |
| new_pron[-1] = "aʊ" |
| continue |
| elif p in ["ɪ", "j"] and len(new_pron) and new_pron[-1] in {"a", "ɑ", "ʌ"}: |
| new_pron[-1] = "aɪ" |
| continue |
| elif p in {"ʊ", "u"} and len(new_pron) and new_pron[-1] in {"ə", "o", "ɔ"}: |
| new_pron[-1] = "oʊ" |
| continue |
| elif p in {"ɹ", "ɚ"} and len(new_pron) and new_pron[-1] in {"o", "ɔ"}: |
| new_pron[-1] = "ɔ" |
| p = "ɹ" |
| elif p in {"ɹ", "ɚ"} and len(new_pron) and new_pron[-1] in {"i", "ɪː", "ɪ"}: |
| new_pron[-1] = "ɪ" |
| p = "ɹ" |
| elif p in {"ɹ", "ɚ"} and len(new_pron) and new_pron[-1] in {"u", "ʊ"}: |
| new_pron[-1] = "ʊ" |
| p = "ɹ" |
| elif p in {"ɹ", "ɚ"} and len(new_pron) and new_pron[-1] in {"e", "ɛ", "ɛː", "æ", "æː"}: |
| new_pron[-1] = "ɛ" |
| p = "ɹ" |
| elif p == "ɹ" and len(new_pron) and new_pron[-1] in ["ɜ", "ɜː"]: |
| new_pron[-1] = "ɝ" |
| continue |
| elif ( |
| p == "ɹ" |
| and len(new_pron) > 1 |
| and new_pron[-1] == "ə" |
| and new_pron[-2] in {"ɪ", "i", "ɪː"} |
| ): |
| new_pron[-1] = "ɹ" |
| new_pron[-2] = "ɪ" |
| continue |
| elif ( |
| p == "ɹ" |
| and len(new_pron) > 1 |
| and new_pron[-1] == "ə" |
| and new_pron[-2] in {"ʊ", "u"} |
| ): |
| new_pron[-1] = "ɹ" |
| new_pron[-2] = "ʊ" |
| continue |
| elif ( |
| p == "ɹ" |
| and len(new_pron) > 1 |
| and new_pron[-1] == "ə" |
| and new_pron[-2] in {"e", "ɛ", "ɛː"} |
| ): |
| new_pron[-1] = "ɹ" |
| new_pron[-2] = "ɛ" |
| continue |
| elif p == "w" and len(new_pron) and new_pron[-1] == "h": |
| new_pron[-1] = "w" |
| continue |
| elif p in {"k", "ɡ"} and len(new_pron) and new_pron[-1] == "n": |
| new_pron[-1] = "ŋ" |
| continue |
| elif p in {"ɜ", "ɜː"} and (i == len(phones) - 1 or phones[i + 1] != "ɹ"): |
| p = "ɝ" |
| elif p == "ɪ" and i == len(phones) - 1: |
| p = "i" |
| elif ( |
| p == "l" and len(new_pron) and new_pron[-1] == "ə" and i == len(phones) - 1 |
| ): |
| new_pron[-1] = "l̩" |
| continue |
| elif ( |
| p == "m" and len(new_pron) and new_pron[-1] == "ə" and i == len(phones) - 1 |
| ): |
| new_pron[-1] = "m̩" |
| continue |
| elif ( |
| p == "n" and len(new_pron) and new_pron[-1] == "ə" and i == len(phones) - 1 |
| ): |
| new_pron[-1] = "n̩" |
| continue |
| elif ( |
| p == "ɹ" and len(new_pron) and new_pron[-1] == "ə" and i == len(phones) - 1 |
| ): |
| new_pron[-1] = "ɚ" |
| continue |
| elif p == "ŋ" and i == len(phones) - 1 and new_pron[-1] == "i": |
| new_pron[-1] = "ɪ" |
| continue |
| elif lang == "english_uk": |
| if lang in LANG_MAPPING: |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| if p == "ɹ" and i == len(phones) - 1: |
| continue |
| elif p == "ɪ" and i == len(phones) - 1: |
| p = "i" |
| elif ( |
| p in {"l", "m", "n"} |
| and i == len(phones) - 1 |
| and len(new_pron) |
| and new_pron[-1] in {"ə", "əː"} |
| ): |
| new_pron[-1] = p + "̩" |
| continue |
| elif p == "ɪ" and len(new_pron) and new_pron[-1] in {"e", "a", "ɔ", "o"}: |
| new_pron[-1] = new_pron[-1] + p |
| continue |
| elif p == "ʊ" and len(new_pron) and new_pron[-1] in {"e", "a"}: |
| new_pron[-1] = new_pron[-1] + p |
| continue |
| elif p in {"ʊ", "u"} and len(new_pron) and new_pron[-1] in {"ə", "o", "ɔ"}: |
| new_pron[-1] = "oʊ" |
| continue |
| elif ( |
| p == "ʒ" and len(new_pron) and new_pron[-1] == "d" |
| ): |
| new_pron[-1] = "dʒ" |
| continue |
| elif ( |
| p == "ʃ" and len(new_pron) and new_pron[-1] == "t" |
| ): |
| new_pron[-1] = "tʃ" |
| continue |
| elif p == "w" and len(new_pron) and new_pron[-1] == "h": |
| new_pron[-1] = "w" |
| continue |
| elif p in {"k", "ɡ"} and len(new_pron) and new_pron[-1] == "n": |
| new_pron[-1] = "ŋ" |
| continue |
| elif p == "ə" and len(new_pron) and new_pron[-1] == "ɛ": |
| new_pron[-1] = "ɛː" |
| continue |
| elif p == "ŋ" and i == len(phones) - 1 and new_pron[-1] == "i": |
| new_pron[-1] = "ɪ" |
| continue |
| elif ( |
| p == "ɹ" |
| and len(new_pron) > 2 |
| and new_pron[-1] == "ə" |
| and new_pron[-2] in {"e", "ɛ", "ʊ", "ɪ", "ɪː", "ɛː"} |
| ): |
| new_pron[-1] = p |
| continue |
| elif lang == "bulgarian": |
| if p in {"s", "ʃ", "sʲ"} and len(new_pron) and new_pron[-1] == "t": |
| new_pron[-1] += p |
| continue |
| elif p == "ʒ" and len(new_pron) and new_pron[-1] == "d": |
| new_pron[-1] = "dʒ" |
| continue |
| elif p in {"ɡ", "k"} and len(new_pron) and new_pron[-1] in {"n"}: |
| new_pron[-1] = "ŋ" |
| elif p in {"v", "f"} and len(new_pron) and new_pron[-1] in {"n"}: |
| new_pron[-1] = "ɱ" |
| elif lang == "czech": |
| if p in {"u", "ʊ"} and len(new_pron) and new_pron[-1] in {"o", "ɔ"}: |
| new_pron[-1] = "ow" |
| continue |
| elif p in ["u", "ʊ"] and len(new_pron) and new_pron[-1] in {"a"}: |
| new_pron[-1] = "aw" |
| continue |
| elif p in {"u", "ʊ"} and len(new_pron) and new_pron[-1] in {"e", "ɛ"}: |
| new_pron[-1] = "ew" |
| continue |
| elif p in {"ʃ", "s"} and len(new_pron) and new_pron[-1] in {"t"}: |
| new_pron[-1] += p |
| continue |
| elif p in {"ʒ"} and len(new_pron) and new_pron[-1] in {"d"}: |
| new_pron[-1] += p |
| continue |
| elif p == "ʊ": |
| p = "u" |
| elif p == "e": |
| p = "ɛ" |
| elif lang.startswith("serbocroatian"): |
| if p in {"ɕ", "ʂ", "ʃ"} and len(new_pron) and new_pron[-1] == "t": |
| new_pron[-1] += p |
| continue |
| elif p in {"ʑ", "ʐ", "ʒ"} and len(new_pron) and new_pron[-1] == "d": |
| new_pron[-1] += p |
| continue |
| elif lang == "german": |
| if lang in LANG_MAPPING: |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| if p in {"ʏ", "ɪ"} and len(new_pron) and new_pron[-1] in {"o", "ɔ"}: |
| new_pron[-1] = "ɔʏ" |
| continue |
| elif p == "ɪ" and len(new_pron) and new_pron[-1] == "a": |
| new_pron[-1] = "aɪ" |
| continue |
| elif p == "ɪ" and len(new_pron) and new_pron[-1] == "ʊ": |
| new_pron[-1] = "ʊɪ" |
| continue |
| elif p == "ʊ" and len(new_pron) and new_pron[-1] == "a": |
| new_pron[-1] = "aʊ" |
| continue |
| elif p == "e" and len(new_pron) and new_pron[-1] == "ɐ": |
| new_pron[-1] = "ɐ" |
| continue |
| elif p == "ʔ": |
| continue |
| elif p in {"tʰ", "kʰ", "pʰ"} and i == len(phones) - 1: |
| p = p[0] |
| elif ( |
| p in {"tʰ", "kʰ", "pʰ"} |
| and len(new_pron) |
| and new_pron[-1] in {"s", "ts", "ʃ", "tʃ"} |
| ): |
| p = p[0] |
| elif p in {"t", "k", "p"} and i == 0: |
| p += "ʰ" |
| elif p in {"s", "ʃ"} and i == 1 and new_pron[-1] in {"tʰ"}: |
| new_pron[-1] = "t" + p |
| continue |
| elif ( |
| p in {"v", "s", "x", "ʁ", "l", "j"} |
| and len(new_pron) |
| and new_pron[-1] in {"tʰ", "kʰ", "pʰ"} |
| ): |
| new_pron[-1] = new_pron[-1][0] |
| elif p == "s" and len(new_pron) and new_pron[-1] == "t": |
| if "z" in word or "c" in word: |
| new_pron[-1] = "ts" |
| continue |
| elif p == "õ": |
| new_pron.append("ɔ") |
| new_pron.append("n") |
| continue |
| elif p == "ɛ̃": |
| new_pron.append("eː") |
| new_pron.append("n") |
| continue |
| elif lang.startswith("mandarin_hani"): |
| vowel_pattern = re.compile(r"^[ayeiouəɚʊɤ̃]+[²³⁰¹⁴⁵]*$") |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| if p in {"²", "³", "¹", "⁰", "⁴", "⁵", "⁻", "⁽", "⁾"} and len(new_pron): |
| index = -1 |
| for j in range(len(new_pron) - 1, -1, -1): |
| if vowel_pattern.match(new_pron[j]) or "̩" in new_pron[j]: |
| index = j |
| break |
| if new_pron[index].endswith("²¹⁴"): |
| continue |
| new_pron[index] += p |
| continue |
| elif p.startswith("ˀ"): |
| new_pron.append("ʔ") |
| if p[1] in LANG_MAPPING[lang]: |
| new_pron.append(LANG_MAPPING[lang][p[1]]) |
| else: |
| new_pron.append(p[1]) |
| continue |
| elif ( |
| any(p.startswith(x) for x in VOWELS[lang]) |
| and len(new_pron) |
| and re.match(r"^[ayeiouəɚʊɤ̃]+$", new_pron[-1]) |
| ): |
| new_pron[-1] += p |
| continue |
| elif lang == "portuguese_brazil": |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| if p == "w̃" and len(new_pron) and new_pron[-1] in {"ɐ̃", "õ"}: |
| new_pron[-1] += p |
| continue |
| elif p == "j̃" and len(new_pron) and new_pron[-1] in {"ɐ̃", "õ", "ẽ", "ũ"}: |
| new_pron[-1] += p |
| continue |
| elif ( |
| p == "j" and len(new_pron) and new_pron[-1] in {"a", "ɛ", "e", "ɐ", "ɔ", "o", "u"} |
| ): |
| new_pron[-1] += p |
| continue |
| elif p == "w" and len(new_pron) and new_pron[-1] in {"a", "ɛ", "e", "ɐ", "i"}: |
| new_pron[-1] += p |
| continue |
| elif lang == "portuguese_portugal": |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| if p == "w̃" and len(new_pron) and new_pron[-1] in {"ɐ̃", "õ"}: |
| new_pron[-1] += p |
| continue |
| elif p == "j̃" and len(new_pron) and new_pron[-1] in {"ɐ̃", "õ", "ẽ", "ũ"}: |
| new_pron[-1] += p |
| continue |
| elif ( |
| p == "j" and len(new_pron) and new_pron[-1] in {"a", "ɛ", "e", "ɐ", "ɔ", "o", "u"} |
| ): |
| new_pron[-1] += p |
| continue |
| elif p == "w" and len(new_pron) and new_pron[-1] in {"a", "ɛ", "e", "ɐ", "i"}: |
| new_pron[-1] += p |
| continue |
| elif lang == "swedish": |
| if p == "ʒ" and len(new_pron) and new_pron[-1] == "d": |
| new_pron[-1] += p |
| continue |
| elif ( |
| p in {"k", "kʰ", "ɡ"} |
| and len(new_pron) |
| and new_pron[-1] in {"m", "n", "ɳ", "ɲ", "ŋ"} |
| ): |
| new_pron[-1] = "ŋ" |
| elif ( |
| p in {"t", "tʰ", "d"} |
| and len(new_pron) |
| and new_pron[-1] in {"m", "n", "ɳ", "ɲ", "ŋ"} |
| ): |
| new_pron[-1] = "n" |
| elif ( |
| p in {"p", "pʰ", "b"} |
| and len(new_pron) |
| and new_pron[-1] in {"m", "n", "ɳ", "ɲ", "ŋ"} |
| ): |
| new_pron[-1] = "m" |
| elif ( |
| p in {"ʈ", "ʈʰ", "ɖ", "ʂ"} |
| and len(new_pron) |
| and new_pron[-1] in {"m", "n", "ɳ", "ɲ", "ŋ"} |
| ): |
| new_pron[-1] = "ɳ" |
| elif p == "s" and len(new_pron) and new_pron[-1] == "r" and "rr" not in word: |
| new_pron[-1] = "ʂ" |
| continue |
| elif p in {"t", "ʈ"} and len(new_pron) and new_pron[-1] == "r" and "rr" not in word: |
| new_pron[-1] = "ʈ" |
| continue |
| elif p in {"d", "ɖ"} and len(new_pron) and new_pron[-1] == "r" and "rr" not in word: |
| new_pron[-1] = "ɖ" |
| continue |
| elif ( |
| p in {"n", "ɳ"} == "n" |
| and len(new_pron) |
| and new_pron[-1] == "r" |
| and "rr" not in word |
| ): |
| new_pron[-1] = "ɳ" |
| continue |
| elif p in {"l", "ɭ"} and len(new_pron) and new_pron[-1] == "r" and "rr" not in word: |
| new_pron[-1] = "ɭ" |
| continue |
| elif p in {"tʰ", "ʈʰ"} and len(new_pron) and new_pron[-1] == "r" and "rr" not in word: |
| new_pron[-1] = "ʈʰ" |
| continue |
| elif p in {"s", "ʂ"} and len(new_pron) and new_pron[-1] == "r" and "rr" not in word: |
| new_pron[-1] = "ʂ" |
| continue |
| elif p == "aʊ": |
| new_pron.append("a") |
| new_pron.append("ʊ") |
| continue |
| elif p in {"r", "n", "l", "t", "k", "ɡ"} and len(new_pron) and new_pron[-1] == "ə": |
| new_pron[-1] = "ɛ" |
| elif p in {"t", "k", "p", "ʈ"} and not len(new_pron): |
| p += "ʰ" |
| elif ( |
| not VOWEL_PATTERNS[lang].match(p) |
| and len(new_pron) |
| and new_pron[-1] in {"tʰ", "kʰ", "pʰ", "ʈʰ"} |
| ): |
| print(new_pron[-1], p) |
| new_pron[-1] = new_pron[-1][0] |
| elif ( |
| p in {"tʰ", "kʰ", "pʰ", "ʈʰ"} |
| and len(new_pron) |
| and (new_pron[-1] in {"ʂ", "s"} or i == len(phones) - 1) |
| ): |
| p = p[0] |
| elif p == "ə" and i == len(phones) - 1: |
| p = "e" |
| elif p in {"r"} and len(new_pron) and new_pron[-1] == "ɜ": |
| new_pron[-1] = "æː" |
| elif p == "ɜ" and i == len(phones) - 1: |
| p = "e" |
| elif lang == "tamil": |
| if p in {"ʊ", "ɪ"} and len(new_pron) and new_pron[-1] == "a": |
| new_pron[-1] += p |
| continue |
| elif lang in ["spanish_spain", "spanish_latin_america"]: |
| if p in {"n", "m", "ɲ"} and len(new_pron) and new_pron[-1] in {"n", "m", "ɲ"}: |
| new_pron[-1] = p |
| continue |
| if p in {"s", "z"} and len(new_pron) and new_pron[-1] in {"s", "z"}: |
| new_pron[-1] = p |
| continue |
| if p in {"x", "k", "ɡ"} and len(new_pron) and new_pron[-1] == "n": |
| new_pron[-1] = "ŋ" |
| elif ( |
| p in {"ɟʝ", "ʝ", "j", "tʃ", "i", "ĩ"} and len(new_pron) and new_pron[-1] in {"n"} |
| ): |
| new_pron[-1] = "ɲ" |
| elif p in {"j", "i", "ĩ", "e", "ẽ"} and len(new_pron) and new_pron[-1] in {"k"}: |
| new_pron[-1] = "c" |
| elif p in {"j", "i", "ĩ", "e", "ẽ"} and len(new_pron) and new_pron[-1] in {"x"}: |
| new_pron[-1] = "ç" |
| elif p in {"j", "i", "ĩ", "e", "ẽ"} and len(new_pron) and new_pron[-1] in {"ɡ"}: |
| new_pron[-1] = "ɟ" |
| elif p in {"j", "i", "ĩ", "e", "ẽ"} and len(new_pron) and new_pron[-1] in {"ɣ"}: |
| new_pron[-1] = "ʝ" |
| elif ( |
| p in {"ɟʝ", "ʝ", "j", "tʃ", "i", "ĩ"} and len(new_pron) and new_pron[-1] in {"l"} |
| ): |
| new_pron[-1] = "ʎ" |
| elif ( |
| p |
| in { |
| "β", |
| "b", |
| "p", |
| } |
| and len(new_pron) |
| and new_pron[-1] == "n" |
| ): |
| new_pron[-1] = "m" |
| elif ( |
| p |
| in { |
| "f", |
| "v", |
| } |
| and len(new_pron) |
| and new_pron[-1] in {"n", "m", "n̪"} |
| ): |
| new_pron[-1] = "ɱ" |
| elif lang == "thai": |
| if p in {"a"} and len(new_pron) and new_pron[-1] in {"i", "iː", "ɯ", "ɯː", "u", "uː"}: |
| new_pron[-1] += p |
| continue |
| elif lang == "turkish": |
| if p == "ʃ" and len(new_pron) and new_pron[-1] in {"t"}: |
| new_pron[-1] += p |
| continue |
| elif p == "ʒ" and len(new_pron) and new_pron[-1] in {"d"}: |
| new_pron[-1] += p |
| continue |
| elif p in {"e", "i", "œ", "y"} and len(new_pron) and new_pron[-1] in {"k"}: |
| new_pron[-1] = "c" |
| elif p in {"e", "i", "œ", "y"} and len(new_pron) and new_pron[-1] in {"ɡ"}: |
| new_pron[-1] = "ɟ" |
| elif p in {"a", "ɯ", "o", "u"} and len(new_pron) and new_pron[-1] in {"l"}: |
| new_pron[-1] = "ɫ" |
| elif p in {"i", "e", "œ", "y"} and len(new_pron) and new_pron[-1] in {"ɫ"}: |
| new_pron[-1] = "l" |
| elif lang == "portuguese_brazil": |
| if p == "ʃ" and len(new_pron) and new_pron[-1] in {"t"}: |
| new_pron[-1] += p |
| continue |
| elif p == "ʒ" and len(new_pron) and new_pron[-1] in {"d"}: |
| new_pron[-1] += p |
| continue |
| elif lang == "russian": |
| voiced_set = { |
| "v", |
| "bʲ", |
| "b", |
| "bː", |
| "d", |
| "dz", |
| "dzʲ", |
| "dʐ", |
| "dʲ", |
| "dʲː", |
| "dː", |
| "v", |
| "vʲ", |
| "vʲː", |
| "vː", |
| "z", |
| "zʲ", |
| "zʲː", |
| "zː", |
| "ɡ", |
| "ɡʲ", |
| "ɡː", |
| "ɣ", |
| "ʐ", |
| "ʐː", |
| "ʑː", |
| } |
| if p in {"ʔ"}: |
| continue |
|
|
| elif lang == "japanese": |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| if p in {".", "˕", "}", "˦˨˦", "˥", "˨˩", "˧", "꜔", "˩", "ʔ", "%", "˩˥"}: |
| continue |
| elif p in { |
| "̥", |
| "̥̥", |
| } and len(new_pron): |
| new_pron[-1] += "̥" |
| continue |
| elif p in {"ᵝ̥"} and len(new_pron): |
| if "̥" not in new_pron[-1] and "ː" not in new_pron[-1]: |
| new_pron[-1] += "̥" |
| continue |
| elif p in {"ː̥"} and len(new_pron): |
| new_pron[-1] += "ː" |
| continue |
| elif ( |
| p in {"j"} |
| and len(new_pron) |
| and new_pron[-1] in {"ɾ", "p", "m", "b", "k", "t", "d", "ç", "ɡ"} |
| ): |
| new_pron[-1] += "ʲ" |
| continue |
| elif p in {"h"} and len(new_pron) and new_pron[-1] in {"c"}: |
| new_pron[-1] = "tɕ" |
| continue |
| elif p in {"p", "pʲ"} and len(new_pron) and new_pron[-1] in {"p̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"b"} and len(new_pron) and new_pron[-1] in {"b̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"ɾ", "ɾʲ"} and len(new_pron) and new_pron[-1] in {"ɾ̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"k", "kʲ"} and len(new_pron) and new_pron[-1] in {"k̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"t", "tʲ"} and len(new_pron) and new_pron[-1] in {"ʔ̥", "t̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"tɕ", "ts"} and len(new_pron) and new_pron[-1] in {"t̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"ɡ"} and len(new_pron) and new_pron[-1] in {"ɡ̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"d", "dz", "ʑ", "dʑ"} and len(new_pron) and new_pron[-1] in {"d̚"}: |
| new_pron[-1] = p + "ː" |
| continue |
| elif p in {"i", "iː", "i̥"} and len(new_pron) and "ʲ" in new_pron[-1]: |
| if len(new_pron) > 2 and "ʲ" in new_pron[-2]: |
| new_pron[-2] = new_pron[-2].replace("ʲ", "") |
|
|
| new_pron[-1] = new_pron[-1].replace("ʲ", "") |
| elif p in {"i"} and len(new_pron) and new_pron[-1] == "n": |
| new_pron[-1] = "ɲ" |
| elif False and p in {"k", "ɡ"} and len(new_pron) and new_pron[-1] == "ɲ": |
| new_pron[-1] = "ŋ" |
| elif False and p in {"t", "d"} and len(new_pron) and new_pron[-1] == "ɲ": |
| new_pron[-1] = "n" |
| elif p in {"dz", "dʑ"} and len(new_pron) and new_pron[-1] not in {"ɲ", "n"}: |
| p = p[1] |
| elif p in { |
| "ɯ̟̃ᵝ", |
| "ɯ̟̊ᵝ", |
| "ɯ̟ᵝː", |
| "ɯ̟ᵝ", |
| "ɨ̥ᵝ", |
| "ɨᵝ", |
| "ɨ̃ᵝ", |
| "ɨᵝː", |
| "ɨ̥", |
| "ɨ̥ː", |
| "ɯ̥ː", |
| "ɯ̥", |
| }: |
| if len(new_pron) and new_pron[-1] in { |
| "t", |
| "tː", |
| "s", |
| "sː", |
| "z", |
| "zː", |
| "ɲː", |
| "ɲ", |
| "ç", |
| "çː", |
| "n", |
| "nː", |
| "ts", |
| "tsː", |
| "ɕ", |
| "tɕ", |
| "tɕː", |
| "ʑ", |
| "ɕː", |
| "ʑː", |
| "ɡʲ", |
| "ɡʲː", |
| "kʲ", |
| "kʲː", |
| "bʲ", |
| "bʲː", |
| "pʲ", |
| "pʲː", |
| "mʲ", |
| "mʲː", |
| "ɾʲː", |
| "ɾʲ", |
| "j", |
| }: |
| new_p = "ɨ" |
| else: |
| new_p = "ɯ" |
| if "̥" in p or "̊" in p: |
| new_p += "̥" |
| if "ː" in p: |
| new_p += "ː" |
| p = new_p |
| if len(new_pron) and new_pron[-1] == "n": |
| new_pron[-1] = "ɲ" |
|
|
| elif lang == "korean_hangul": |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| if p == "t͈" and "ᄄ" not in jamo.h2j(word): |
| if len(new_pron) and "̚" in new_pron[-1]: |
| p = "tʰ" |
| else: |
| p = "t" |
| elif p == "tɕ͈" and "ᄍ" not in jamo.h2j(word): |
| if len(new_pron) and "̚" in new_pron[-1]: |
| p = "tɕʰ" |
| else: |
| p = "tɕ" |
| elif p == "k͈" and "ᄁ" not in jamo.h2j(word): |
| if len(new_pron) and "̚" in new_pron[-1]: |
| p = "kʰ" |
| else: |
| p = "k" |
| elif p == "p͈" and "ᄈ" not in jamo.h2j(word): |
| if len(new_pron) and "̚" in new_pron[-1]: |
| p = "pʰ" |
| else: |
| p = "p" |
| elif p == "s͈" and "ᄊ" not in jamo.h2j(word): |
| if len(new_pron) and "̚" in new_pron[-1]: |
| p = "sʰ" |
| else: |
| p = "s" |
| elif p == "x" and len(new_pron) and new_pron[-1] == "k": |
| new_pron[-1] += "ʰ" |
| continue |
|
|
| elif lang in ["vietnamese_hanoi", "vietnamese_hue", "vietnamese_hochiminhcity"]: |
| vowel_pattern = re.compile(rf'^[{"".join(VOWELS[lang])}]+$') |
| tone_pattern = re.compile(r"^[˥˩˦˥˨˧˨˩˩˩ˀ˦]+$") |
| if p in {"j", "w"} and len(new_pron) and vowel_pattern.match(new_pron[-1]): |
| new_pron[-1] += p |
| continue |
| elif vowel_pattern.match(p) and len(new_pron) and vowel_pattern.match(new_pron[-1]): |
| new_pron[-1] += p |
| continue |
| elif p in {"ɗ", "ɓ"} and len(new_pron) and new_pron[-1] == "ʔ": |
| new_pron[-1] = p |
| continue |
| elif ( |
| p == "ʔ" |
| and len(new_pron) |
| and tone_pattern.match(new_pron[-1]) |
| and not (i < len(phones) - 1 and phones[i + 1] in {"ɗ", "ɓ"}) |
| ): |
| new_pron[-1] += "ˀ" |
| continue |
| if lang in LANG_MAPPING: |
| for k, v in LANG_MAPPING[lang].items(): |
| if p == k: |
| p = v |
| break |
| if not p: |
| continue |
| new_pron.append(p) |
| tone_mapping = { |
| "⁰": "", |
| "¹": "˩", |
| "²": "˨", |
| "³": "˧", |
| "⁴": "˦", |
| "⁵": "˥", |
| "˧": "˧", |
| "˨˩": "˨˩", |
| "˥˩": "˥˩", |
| "˦˥": "˦˥", |
| "˩˩˦": "˩˩˦", |
| } |
| if lang == "thai": |
| phones = new_pron |
| new_pron = [] |
| tone_symbols = {"˥˩", "˦˥", "˧", "˨˩", "˩˩˦"} |
| vowel_set = {x for x in VOWELS[lang]} |
| vowel_set |= {x + y for x, y in itertools.product(VOWELS[lang], VOWELS[lang])} |
| vowel_set |= { |
| x + y + z for x, y, z in itertools.product(VOWELS[lang], VOWELS[lang], VOWELS[lang]) |
| } |
| for i, p in enumerate(phones): |
| if p in tone_symbols: |
| for j in range(len(new_pron) - 1, 0, -1): |
| if new_pron[j] in vowel_set and new_pron[j] not in {"w", "j"}: |
| new_pron[j] += tone_mapping[p] |
| break |
| else: |
| new_pron.append(p) |
| |
| |
| |
| |
| |
|
|
| elif lang in ["vietnamese_hanoi", "vietnamese_hue", "vietnamese_hochiminhcity"]: |
| phones = new_pron |
| new_pron = [] |
| vowel_pattern = re.compile(rf'^[{"".join(VOWELS[lang])}]+[wj]?$') |
| tone_symbols = { |
| "˦ˀ˥", |
| "˧˦", |
| "˧˧", |
| "˧˨", |
| "˧˩", |
| "˨˩", |
| "˦˧˥", |
| "˦˩", |
| "˧˧", |
| "˧˨", |
| "˨˩", |
| "˨˩˦", |
| "˦˥", |
| "˨˩˨", |
| } |
| tone_pattern = re.compile(r"^[˥˩˦˥˨˧˨˩˩˩ˀ˦]+$") |
| for i, p in enumerate(phones): |
| if tone_pattern.match(p): |
| for j in range(len(new_pron) - 1, 0, -1): |
| if vowel_pattern.match(new_pron[j]): |
| new_pron[j] += p |
| break |
| else: |
| new_pron.append(p) |
| elif lang.startswith("mandarin_hani"): |
| mapping = { |
| "²¹⁴": "˨˩˦", |
| "⁵⁵": "˥˥", |
| "³⁵": "˧˥", |
| "⁵¹": "˥˩", |
| "⁰": "", |
| "¹": "˩", |
| "²": "˨", |
| "³": "˨", |
| "⁴": "˦", |
| "⁵": "˥", |
| } |
| tone_symbols = {"²", "³", "¹", "⁴", "⁵", "⁰"} |
| for i, p in enumerate(new_pron): |
| if any(x in p for x in tone_symbols): |
| for k, v in mapping.items(): |
| if k in new_pron[i]: |
| new_pron[i] = new_pron[i].replace(k, v) |
| |
| |
| elif lang == "swedish": |
| for i, p in enumerate(new_pron): |
| if p == "êː": |
| new_pron[i] = "eː˧˩" |
| elif p == "â": |
| new_pron[i] = "a˧˩" |
| elif p == "ɛ̂": |
| new_pron[i] = "ɛ˧˩" |
| elif p == "ɑ̂ː": |
| new_pron[i] = "ɑː˧˩" |
| elif p == "ûː": |
| new_pron[i] = "uː˧˩" |
| elif p == "ʉ̂ː": |
| new_pron[i] = "ʉː˧˩" |
| elif p == "ɵ̂": |
| new_pron[i] = "ɵ˧˩" |
| elif p == "ʉ̂ː": |
| new_pron[i] = "ʉː˧˩" |
| elif p == "ʉ̟ː˥˩": |
| new_pron[i] = "ʉː˥˩" |
| elif p == "ǎ": |
| new_pron[i] = "a˥˧" |
| elif p == "ʉ̟ː˧˩": |
| new_pron[i] = "ʉː˧˩" |
| elif p == "ø̀ː": |
| new_pron[i] = "øː˩" |
| elif p == "ɑ̀ː": |
| new_pron[i] = "ɑː˩" |
| elif p == "ỳː": |
| new_pron[i] = "yː˩" |
| elif p == "ỳː˧˩": |
| new_pron[i] = "yː˧˩" |
|
|
| elif lang == "hausa": |
| phone_mapping = { |
| "á": "a", |
| "áː": "aː", |
| "é": "e", |
| "éː": "eː", |
| "í": "i", |
| "íː": "iː", |
| "ó": "o", |
| "óː": "oː", |
| "úː": "uː", |
| "à": "a", |
| "àː": "aː", |
| "è": "e", |
| "èː": "eː", |
| "ì": "i", |
| "ìː": "iː", |
| "ò": "o", |
| "òː": "oː", |
| "ùː": "uː", |
| "â": "a", |
| "âː": "aː", |
| "ê": "e", |
| "êː": "eː", |
| "î": "i", |
| "îː": "iː", |
| "ô": "o", |
| "ôː": "oː", |
| "ûː": "uː", |
| } |
| for i, p in enumerate(new_pron): |
| if p in {"á", "áː", "é", "éː", "í", "íː", "ó", "óː", "úː"} or "́" in p: |
| if p in phone_mapping: |
| new_pron[i] = phone_mapping[p] |
| else: |
| new_pron[i] = p.replace("́", "") |
| new_pron[i] += "˥" |
| elif p in {"à", "àː", "è", "èː", "ì", "ìː", "ò", "òː", "ùː"} or "̀" in p: |
| if p in phone_mapping: |
| new_pron[i] = phone_mapping[p] |
| else: |
| new_pron[i] = p.replace("̀", "") |
| new_pron[i] += "˩" |
| elif ( |
| p |
| in { |
| "â", |
| "âː", |
| "ê", |
| "êː", |
| "î", |
| "îː", |
| "ôː", |
| "ûː", |
| } |
| or "̂" in p |
| ): |
| if p in phone_mapping: |
| new_pron[i] = phone_mapping[p] |
| else: |
| new_pron[i] = p.replace("̂", "") |
| new_pron[i] += "˥˦" |
|
|
| return new_pron |
|
|
|
|
| def convert_second_round(word, phones, lang): |
| if lang not in ["english_us", "english_uk"]: |
| return phones |
| new_pron = [] |
| stressed_vowels = {} |
| if lang == "english_uk": |
| stressed_vowels = { |
| "aɪ", |
| "aʊ", |
| "eɪ", |
| "i", |
| "iː", |
| "oɪ", |
| "oʊ", |
| "u", |
| "uː", |
| "æ", |
| "ɑ", |
| "ɑː", |
| "ɒ", |
| "ɔ", |
| "ɔɪ", |
| "ɔː", |
| "ɛ", |
| "ɛː", |
| "ɜ", |
| "ɜː", |
| "ʊ", |
| "ʌ", |
| } |
| elif lang == "english_us": |
| stressed_vowels = { |
| "aɪ", |
| "aʊ", |
| "eɪ", |
| "i", |
| "iː", |
| "oɪ", |
| "oʊ", |
| "u", |
| "uː", |
| "æ", |
| "ɑ", |
| "ɑː", |
| "ɔ", |
| "ɔɪ", |
| "ɔː", |
| "ɛ", |
| "ɝ", |
| "ɝː", |
| "ʊ", |
| "ʌ", |
| } |
| all_syllabics = {"ɪ", "ə", "ɚ", "n̩", "m̩", "l̩", "ɫ̩"} | VOWELS[lang] |
| for i, p in enumerate(phones): |
| if lang in ["english_us", "english_uk"]: |
| if ( |
| p == "l" |
| and 2 < i < len(phones) - 1 |
| and new_pron[-1] == "ə" |
| and phones[i + 1] not in all_syllabics |
| ): |
| new_pron[-1] = "ɫ̩" |
| continue |
| elif ( |
| p == "m" |
| and 2 < i < len(phones) - 1 |
| and new_pron[-1] == "ə" |
| and phones[i + 1] not in all_syllabics |
| ): |
| new_pron[-1] = "m̩" |
| continue |
| elif ( |
| p == "n" |
| and 2 < i < len(phones) - 1 |
| and new_pron[-1] == "ə" |
| and phones[i + 1] not in all_syllabics |
| ): |
| new_pron[-1] = "n̩" |
| continue |
| elif p == "l̩" and 1 < i < len(phones) - 1 and phones[i + 1] in all_syllabics: |
| new_pron.append("ə") |
| p = "l" |
| elif p == "l̩": |
| p = "ɫ̩" |
| elif p == "l" and i == len(phones) - 1: |
| p = "ɫ" |
| elif p == "l" and 1 < i < len(phones) - 1 and phones[i + 1] not in all_syllabics: |
| p = "ɫ" |
| elif ( |
| p in {"t", "p", "k"} |
| and i == 0 |
| and i < len(phones) - 1 |
| and phones[i + 1] in stressed_vowels | {"ɪ", "ə", "ɚ"} |
| ): |
| p += "ʰ" |
| elif ( |
| p == "ə" |
| and 1 < i == len(phones) - 2 |
| and phones[i - 1] in {"d", "t"} |
| and phones[i + 1] == "d" |
| ): |
| p = "ɪ" |
| elif ( |
| p == "ə" |
| and 1 < i == len(phones) - 2 |
| and phones[i - 1] in {"s", "z", "ʃ", "ʒ", "tʃ", "dʒ"} |
| and phones[i + 1] == "z" |
| ): |
| p = "ɪ" |
| if lang == "english_us": |
| if ( |
| p == "ɹ" |
| and 2 < i < len(phones) - 1 |
| and new_pron[-1] == "ə" |
| and phones[i + 1] not in all_syllabics |
| ): |
| new_pron[-1] = "ɚ" |
| continue |
| elif ( |
| p in {"d", "t"} |
| and 1 < i < len(phones) - 1 |
| and phones[i - 1] in all_syllabics |
| and phones[i + 1] in {"n̩", "m̩", "l̩", "ɚ", "ə", "ɫ̩"} |
| ): |
| p = "ɾ" |
| elif ( |
| p in {"t", "d"} |
| and 1 < i < len(phones) - 2 |
| and phones[i - 1] in all_syllabics |
| and phones[i + 1] == "ɪ" |
| and phones[i + 2] == "d" |
| ): |
| p = "ɾ" |
| elif ( |
| p in {"t", "d"} |
| and i > 1 |
| and i == len(phones) - 2 |
| and phones[i - 1] in all_syllabics |
| and phones[i + 1] == "i" |
| ): |
| p = "ɾ" |
| elif ( |
| p in {"t", "d"} |
| and i > 1 |
| and i == len(phones) - 3 |
| and phones[i - 1] in all_syllabics |
| and phones[i + 1] in {"i", "ɪ"} |
| and phones[i + 2] == "z" |
| ): |
| p = "ɾ" |
| elif ( |
| p in {"t", "d"} |
| and i > 1 |
| and i == len(phones) - 3 |
| and phones[i - 1] in all_syllabics |
| and phones[i + 1] == "ɪ" |
| and phones[i + 2] == "ŋ" |
| ): |
| p = "ɾ" |
| elif ( |
| p in {"t", "p", "k"} |
| and i == 0 |
| and i < len(phones) - 1 |
| and phones[i + 1] in stressed_vowels | {"ɪ", "ə", "ɚ"} |
| ): |
| p += "ʰ" |
| elif ( |
| p in {"t", "p", "k"} |
| and i > 0 |
| and phones[i - 1] not in {"s", "ʃ"} |
| and i < len(phones) - 1 |
| and phones[i + 1] in stressed_vowels |
| ): |
| p += "ʰ" |
| elif p == "l̩" and 1 < i < len(phones) - 1 and phones[i + 1] in all_syllabics: |
| new_pron.append("ə") |
| p = "l" |
| elif p == "l̩": |
| p = "ɫ̩" |
| elif p == "l" and i == len(phones) - 1: |
| p = "ɫ" |
| elif ( |
| p == "l" |
| and 1 < i < len(phones) - 1 |
| and phones[i + 1] not in {"ɪ", "ə", "ɚ", "n̩", "m̩", "l̩", "ɫ̩"} | VOWELS[lang] |
| ): |
| p = "ɫ" |
| elif ( |
| p == "ə" |
| and 1 < i == len(phones) - 2 |
| and phones[i - 1] in {"d", "t", "ɾ"} |
| and phones[i + 1] == "d" |
| ): |
| p = "ɪ" |
| elif ( |
| p == "ə" |
| and 1 < i == len(phones) - 2 |
| and phones[i - 1] in {"s", "z", "ʃ", "ʒ", "tʃ", "dʒ"} |
| and phones[i + 1] == "z" |
| ): |
| p = "ɪ" |
| elif ( |
| p in {"t", "p", "k"} |
| and i > 0 |
| and phones[i - 1] not in {"s", "ʃ"} |
| and i < len(phones) - 1 |
| and phones[i + 1] in stressed_vowels |
| ): |
| p += "ʰ" |
| elif lang == "english_uk": |
| if p not in VOWELS[lang] and len(new_pron) and new_pron[-1] == "ɹ": |
| new_pron[-1] = p |
| continue |
| elif ( |
| p in {"t", "p", "k"} |
| and i > 0 |
| and phones[i - 1] not in {"s", "ʃ"} |
| and i < len(phones) - 1 |
| and phones[i + 1] in stressed_vowels |
| ): |
| p += "ʰ" |
| new_pron.append(p) |
| return new_pron |
|
|
|
|
| def fix_pronunciations(dictionary, lang): |
| filtered_dictionary = [] |
| for word, pronunciation in dictionary: |
| if lang == "polish": |
| if "ü" in word: |
| continue |
| for i, p in enumerate(pronunciation): |
| if p in LANG_MAPPING[lang]: |
| continue |
| if p in GLOBAL_REMAPPING: |
| pronunciation[i] = GLOBAL_REMAPPING[p] |
| elif "̯" in p: |
| pronunciation[i] = p.replace("̯", "") |
| elif "͡" in p: |
| pronunciation[i] = p.replace("͡", "") |
| elif "‿" in p: |
| pronunciation[i] = p.replace("‿", "") |
| elif "͜" in p: |
| pronunciation[i] = p.replace("͜", "") |
| elif "g" in p: |
| pronunciation[i] = p.replace("g", "ɡ") |
| |
| new_pron = convert_language_specific(word, pronunciation, lang) |
| new_pron = convert_second_round(word, new_pron, lang) |
| if new_pron is None: |
| continue |
| if (word, new_pron) not in filtered_dictionary: |
| filtered_dictionary.append((word, new_pron)) |
| return filtered_dictionary |
|
|
|
|
| def process_language(lang): |
| print("Processing", lang) |
| if lang == "japanese": |
| dictionary, input_graphemes, input_phones = read_source(lang + "_hiragana") |
| d, g, p = read_source(lang + "_katakana") |
| dictionary.extend(d) |
| input_graphemes.update(g) |
| input_phones.update(p) |
| word_set = {x[0] for x in dictionary} |
| d, g, p = read_source(lang) |
| dictionary.extend([x for x in d if x[0] not in word_set]) |
| input_graphemes.update(g) |
| input_phones.update(p) |
|
|
| else: |
| dictionary, input_graphemes, input_phones = read_source(lang) |
|
|
| print("Input graphemes", sorted(input_graphemes)) |
| print("Input phones", sorted(input_phones)) |
| filtered = fix_pronunciations(dictionary, lang) |
| save_dictionary(filtered, lang) |
|
|
|
|
| if __name__ == "__main__": |
| for code in LANG_CODES: |
| process_language(code) |
|
|
