| """英文文本归一化 (TN): 数字 / 货币 / 百分比 / 日期 / 年份 / 电话号码等 NSW -> 英文单词。 |
| |
| [导出包 vendored 版本] 与 bridge/text_norm.py 逻辑一致, FST 路径相对模型根目录 |
| (env DUBBING_HOME 或 DUBBING_WETEXT 覆盖, 缺省取本文件上三级/wetext_en_tn)。 |
| |
| 规则版本: wetext-0.0.4-en-tn-v1。实现: wetext 0.0.4 en TN (kaldifst FST, 本地文件, |
| 不触网); wetext 不可用时回退 inflect 手写规则 (仅兜底, 打 warning)。 |
| """ |
| import logging |
| import os |
| import re |
|
|
| RULE_VERSION = 'wetext-0.0.4-en-tn-v1' |
|
|
| _MODEL_DIR = os.environ.get('DUBBING_HOME') or os.path.abspath( |
| os.path.join(os.path.dirname(__file__), '..', '..')) |
|
|
| |
| WETEXT_EN_TN = os.environ.get('DUBBING_WETEXT') or os.path.join(_MODEL_DIR, 'wetext_en_tn') |
|
|
| _NORMALIZER = None |
| _BACKEND = None |
|
|
| |
| _RE_SPECIAL = re.compile(r'<\|[^|]*\|>') |
| _RE_TS_BRACKET = re.compile(r'[\(\[]\s*\d{1,2}:\d{2}(?::\d{2})?(?:\.\d+)?\s*[\)\]]') |
| _RE_WS = re.compile(r'\s+') |
|
|
|
|
| def _get_backend(): |
| """惰性加载 wetext en TN (本地 FST); 失败回退 inflect 规则。返回 'wetext'/'inflect'。""" |
| global _NORMALIZER, _BACKEND |
| if _BACKEND is None: |
| try: |
| from wetext import Normalizer |
| tagger = os.path.join(WETEXT_EN_TN, 'tagger.fst') |
| verbalizer = os.path.join(WETEXT_EN_TN, 'verbalizer.fst') |
| if not (os.path.exists(tagger) and os.path.exists(verbalizer)): |
| raise FileNotFoundError(WETEXT_EN_TN) |
| _NORMALIZER = Normalizer(tagger_path=tagger, verbalizer_path=verbalizer, lang='en') |
| _BACKEND = 'wetext' |
| except Exception as e: |
| logging.warning(f'text_norm: wetext unavailable ({type(e).__name__}: {e}), ' |
| f'fallback to inflect rules') |
| _BACKEND = 'inflect' |
| return _BACKEND |
|
|
|
|
| _MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july', |
| 'august', 'september', 'october', 'november', 'december'] |
|
|
|
|
| def _inflect_normalize(text): |
| """兜底规则 (无 wetext 时): 货币 / 百分比 / ISO 日期 / 序数 / 小数 / 整数。子集, 仅兜底。""" |
| import inflect |
| p = inflect.engine() |
|
|
| def int_words(s): |
| return p.number_to_words(int(s)).replace('-', ' ').replace(',', '') |
|
|
| def num_words(s): |
| if '.' in s: |
| i, f = s.split('.', 1) |
| head = int_words(i) if i and int(i) else 'zero' |
| return head + ' point ' + ' '.join(int_words(d) for d in f) |
| return int_words(s) |
|
|
| cur = {'$': 'dollars', '€': 'euros', '£': 'pounds'} |
| text = re.sub(r'([$€£])\s*(\d+(?:\.\d+)?)', |
| lambda m: num_words(m.group(2)) + ' ' + cur[m.group(1)], text) |
| text = re.sub(r'(\d+(?:\.\d+)?)\s*%', |
| lambda m: num_words(m.group(1)) + ' percent', text) |
| text = re.sub(r'\b(\d{4})-(\d{2})-(\d{2})\b', |
| lambda m: f"the {p.ordinal(int_words(m.group(3)))} of " |
| f"{_MONTHS[int(m.group(2)) - 1]} {int_words(m.group(1))}", text) |
| text = re.sub(r'\b(\d+)(st|nd|rd|th)\b', lambda m: p.ordinal(int_words(m.group(1))), text) |
| text = re.sub(r'\b\d+\.\d+\b', lambda m: num_words(m.group(0)), text) |
| text = re.sub(r'\b\d+\b', lambda m: num_words(m.group(0)), text) |
| return text |
|
|
|
|
| def strip_meta(text): |
| """剥掉 ST LM 输出里可能夹带的特殊 token / 时间戳标记。""" |
| text = _RE_SPECIAL.sub(' ', text) |
| text = _RE_TS_BRACKET.sub(' ', text) |
| return _RE_WS.sub(' ', text).strip() |
|
|
|
|
| def normalize_en(text): |
| """英文译文归一化入口。幂等 (normalize_en(normalize_en(x)) == normalize_en(x))。""" |
| text = strip_meta(text) |
| if not text: |
| return text |
| if _get_backend() == 'wetext': |
| out = _NORMALIZER.normalize(text) |
| else: |
| out = _inflect_normalize(text) |
| return _RE_WS.sub(' ', out).strip() |
|
|
