Luigi commited on
Commit
42e1bbc
·
verified ·
1 Parent(s): 0c0d3c8

frontend: entity-aware normalizer

Browse files
Files changed (1) hide show
  1. scripts/frontend_bopomofo.py +3 -23
scripts/frontend_bopomofo.py CHANGED
@@ -20,33 +20,13 @@ LANG = {'ZH': 0, 'EN': 1} # per-phone language id
20
 
21
  _g2pw = None
22
  _g2pen = None
23
- _zh_num = None
24
-
25
- _ZH_DIGIT = {"0":"零","1":"一","2":"二","3":"三","4":"四","5":"五","6":"六","7":"七","8":"八","9":"九"}
26
-
27
- def _make_zh_normalizer():
28
- import cn2an
29
- def norm(text):
30
- # long digit runs (>=5, e.g. phone/order numbers) -> digit-by-digit zh; else cardinal
31
- def repl(m):
32
- d = m.group(0)
33
- if len(d) >= 5:
34
- return "".join(_ZH_DIGIT[c] for c in d)
35
- try:
36
- return cn2an.an2cn(d, "low")
37
- except Exception:
38
- return "".join(_ZH_DIGIT[c] for c in d)
39
- text = re.sub(r"\d+", repl, text)
40
- text = text.replace(",", ",").replace("。", ".").replace("?", "?").replace("!", "!")
41
- return text
42
- return norm
43
 
44
  def _lazy():
45
- global _g2pw, _g2pen, _zh_num
46
  if _g2pw is None:
47
  _g2pw = G2PWConverter()
48
  _g2pen = G2p()
49
- _zh_num = _make_zh_normalizer()
50
 
51
  def _split_syllable(syl: str):
52
  """'ㄓㄨㄢ3' -> (['ㄓ','ㄨ','ㄢ'], tone 3)."""
@@ -58,7 +38,7 @@ def _split_syllable(syl: str):
58
 
59
  def text_to_phones(text: str):
60
  _lazy()
61
- text = _zh_num(text) # numbers->zh words, normalize punct
62
  bopo = _g2pw(text)[0] # per-char bopomofo or None
63
  chars = list(text)
64
  phones, tones, langs = [], [], []
 
20
 
21
  _g2pw = None
22
  _g2pen = None
23
+ import text_norm # entity-aware normalizer (phone/email/price/date/…)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
 
25
  def _lazy():
26
+ global _g2pw, _g2pen
27
  if _g2pw is None:
28
  _g2pw = G2PWConverter()
29
  _g2pen = G2p()
 
30
 
31
  def _split_syllable(syl: str):
32
  """'ㄓㄨㄢ3' -> (['ㄓ','ㄨ','ㄢ'], tone 3)."""
 
38
 
39
  def text_to_phones(text: str):
40
  _lazy()
41
+ text = text_norm.normalize(text) # entities -> spoken form, normalize punct
42
  bopo = _g2pw(text)[0] # per-char bopomofo or None
43
  chars = list(text)
44
  phones, tones, langs = [], [], []