"""英文文本归一化 (TN): 数字 / 货币 / 百分比 / 日期 / 年份 / 电话号码等 NSW -> 英文单词。 [导出包 vendored 版本] 与 bridge/text_norm.py 逻辑一致, FST 路径相对模型根目录 (env DUBBING_HOME 或 DUBBING_WETEXT 覆盖, 缺省取本文件上三级/wetext_en_tn)。 规则版本: wetext-0.0.4-en-tn-v1。实现: wetext 0.0.4 en TN (kaldifst FST, 本地文件, 不触网); wetext 不可用时回退 inflect 手写规则 (仅兜底, 打 warning)。 """ import logging import os import re RULE_VERSION = 'wetext-0.0.4-en-tn-v1' _MODEL_DIR = os.environ.get('DUBBING_HOME') or os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) # en TN FST (导出包 wetext_en_tn/; 可用 DUBBING_WETEXT 覆盖) WETEXT_EN_TN = os.environ.get('DUBBING_WETEXT') or os.path.join(_MODEL_DIR, 'wetext_en_tn') _NORMALIZER = None _BACKEND = None # ST LM 译文偶带的元标记: <|...|> 特殊 token、[...]/(...) 时间戳 _RE_SPECIAL = re.compile(r'<\|[^|]*\|>') _RE_TS_BRACKET = re.compile(r'[\(\[]\s*\d{1,2}:\d{2}(?::\d{2})?(?:\.\d+)?\s*[\)\]]') _RE_WS = re.compile(r'\s+') def _get_backend(): """惰性加载 wetext en TN (本地 FST); 失败回退 inflect 规则。返回 'wetext'/'inflect'。""" global _NORMALIZER, _BACKEND if _BACKEND is None: try: from wetext import Normalizer tagger = os.path.join(WETEXT_EN_TN, 'tagger.fst') verbalizer = os.path.join(WETEXT_EN_TN, 'verbalizer.fst') if not (os.path.exists(tagger) and os.path.exists(verbalizer)): raise FileNotFoundError(WETEXT_EN_TN) _NORMALIZER = Normalizer(tagger_path=tagger, verbalizer_path=verbalizer, lang='en') _BACKEND = 'wetext' except Exception as e: logging.warning(f'text_norm: wetext unavailable ({type(e).__name__}: {e}), ' f'fallback to inflect rules') _BACKEND = 'inflect' return _BACKEND _MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december'] def _inflect_normalize(text): """兜底规则 (无 wetext 时): 货币 / 百分比 / ISO 日期 / 序数 / 小数 / 整数。子集, 仅兜底。""" import inflect p = inflect.engine() def int_words(s): return p.number_to_words(int(s)).replace('-', ' ').replace(',', '') def num_words(s): if '.' in s: i, f = s.split('.', 1) head = int_words(i) if i and int(i) else 'zero' return head + ' point ' + ' '.join(int_words(d) for d in f) return int_words(s) cur = {'$': 'dollars', '€': 'euros', '£': 'pounds'} text = re.sub(r'([$€£])\s*(\d+(?:\.\d+)?)', lambda m: num_words(m.group(2)) + ' ' + cur[m.group(1)], text) text = re.sub(r'(\d+(?:\.\d+)?)\s*%', lambda m: num_words(m.group(1)) + ' percent', text) text = re.sub(r'\b(\d{4})-(\d{2})-(\d{2})\b', lambda m: f"the {p.ordinal(int_words(m.group(3)))} of " f"{_MONTHS[int(m.group(2)) - 1]} {int_words(m.group(1))}", text) text = re.sub(r'\b(\d+)(st|nd|rd|th)\b', lambda m: p.ordinal(int_words(m.group(1))), text) text = re.sub(r'\b\d+\.\d+\b', lambda m: num_words(m.group(0)), text) text = re.sub(r'\b\d+\b', lambda m: num_words(m.group(0)), text) return text def strip_meta(text): """剥掉 ST LM 输出里可能夹带的特殊 token / 时间戳标记。""" text = _RE_SPECIAL.sub(' ', text) text = _RE_TS_BRACKET.sub(' ', text) return _RE_WS.sub(' ', text).strip() def normalize_en(text): """英文译文归一化入口。幂等 (normalize_en(normalize_en(x)) == normalize_en(x))。""" text = strip_meta(text) if not text: return text if _get_backend() == 'wetext': out = _NORMALIZER.normalize(text) else: out = _inflect_normalize(text) return _RE_WS.sub(' ', out).strip()