File size: 4,027 Bytes
46fc068 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | """英文文本归一化 (TN): 数字 / 货币 / 百分比 / 日期 / 年份 / 电话号码等 NSW -> 英文单词。
[导出包 vendored 版本] 与 bridge/text_norm.py 逻辑一致, FST 路径相对模型根目录
(env DUBBING_HOME 或 DUBBING_WETEXT 覆盖, 缺省取本文件上三级/wetext_en_tn)。
规则版本: wetext-0.0.4-en-tn-v1。实现: wetext 0.0.4 en TN (kaldifst FST, 本地文件,
不触网); wetext 不可用时回退 inflect 手写规则 (仅兜底, 打 warning)。
"""
import logging
import os
import re
RULE_VERSION = 'wetext-0.0.4-en-tn-v1'
_MODEL_DIR = os.environ.get('DUBBING_HOME') or os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..'))
# en TN FST (导出包 wetext_en_tn/; 可用 DUBBING_WETEXT 覆盖)
WETEXT_EN_TN = os.environ.get('DUBBING_WETEXT') or os.path.join(_MODEL_DIR, 'wetext_en_tn')
_NORMALIZER = None
_BACKEND = None
# ST LM 译文偶带的元标记: <|...|> 特殊 token、[...]/(...) 时间戳
_RE_SPECIAL = re.compile(r'<\|[^|]*\|>')
_RE_TS_BRACKET = re.compile(r'[\(\[]\s*\d{1,2}:\d{2}(?::\d{2})?(?:\.\d+)?\s*[\)\]]')
_RE_WS = re.compile(r'\s+')
def _get_backend():
"""惰性加载 wetext en TN (本地 FST); 失败回退 inflect 规则。返回 'wetext'/'inflect'。"""
global _NORMALIZER, _BACKEND
if _BACKEND is None:
try:
from wetext import Normalizer
tagger = os.path.join(WETEXT_EN_TN, 'tagger.fst')
verbalizer = os.path.join(WETEXT_EN_TN, 'verbalizer.fst')
if not (os.path.exists(tagger) and os.path.exists(verbalizer)):
raise FileNotFoundError(WETEXT_EN_TN)
_NORMALIZER = Normalizer(tagger_path=tagger, verbalizer_path=verbalizer, lang='en')
_BACKEND = 'wetext'
except Exception as e:
logging.warning(f'text_norm: wetext unavailable ({type(e).__name__}: {e}), '
f'fallback to inflect rules')
_BACKEND = 'inflect'
return _BACKEND
_MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july',
'august', 'september', 'october', 'november', 'december']
def _inflect_normalize(text):
"""兜底规则 (无 wetext 时): 货币 / 百分比 / ISO 日期 / 序数 / 小数 / 整数。子集, 仅兜底。"""
import inflect
p = inflect.engine()
def int_words(s):
return p.number_to_words(int(s)).replace('-', ' ').replace(',', '')
def num_words(s):
if '.' in s:
i, f = s.split('.', 1)
head = int_words(i) if i and int(i) else 'zero'
return head + ' point ' + ' '.join(int_words(d) for d in f)
return int_words(s)
cur = {'$': 'dollars', '€': 'euros', '£': 'pounds'}
text = re.sub(r'([$€£])\s*(\d+(?:\.\d+)?)',
lambda m: num_words(m.group(2)) + ' ' + cur[m.group(1)], text)
text = re.sub(r'(\d+(?:\.\d+)?)\s*%',
lambda m: num_words(m.group(1)) + ' percent', text)
text = re.sub(r'\b(\d{4})-(\d{2})-(\d{2})\b',
lambda m: f"the {p.ordinal(int_words(m.group(3)))} of "
f"{_MONTHS[int(m.group(2)) - 1]} {int_words(m.group(1))}", text)
text = re.sub(r'\b(\d+)(st|nd|rd|th)\b', lambda m: p.ordinal(int_words(m.group(1))), text)
text = re.sub(r'\b\d+\.\d+\b', lambda m: num_words(m.group(0)), text)
text = re.sub(r'\b\d+\b', lambda m: num_words(m.group(0)), text)
return text
def strip_meta(text):
"""剥掉 ST LM 输出里可能夹带的特殊 token / 时间戳标记。"""
text = _RE_SPECIAL.sub(' ', text)
text = _RE_TS_BRACKET.sub(' ', text)
return _RE_WS.sub(' ', text).strip()
def normalize_en(text):
"""英文译文归一化入口。幂等 (normalize_en(normalize_en(x)) == normalize_en(x))。"""
text = strip_meta(text)
if not text:
return text
if _get_backend() == 'wetext':
out = _NORMALIZER.normalize(text)
else:
out = _inflect_normalize(text)
return _RE_WS.sub(' ', out).strip()
|