| """Join dictated single-letter runs before the text model, never after it.""" |
| import re |
|
|
| NORMALIZATION_VERSION = "spelled-letter-runs-v2" |
|
|
| |
| _RUN = re.compile(r'(?<![A-Za-z0-9_./:@+\-])(?>[A-Za-z](?:(?:[-‐‑‒–—]|[ \t\u00a0]+)[A-Za-z])+)(?![A-Za-z0-9_./:@+\-])') |
| _LITERAL = re.compile( |
| r'```[\s\S]*?(?:```|\Z)|`[^`\n]*(?:`|\Z)' |
| r'|https?://[^\s，。！？；、）]+' |
| r'|[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}' |
| r'|(?:~/|/)[A-Za-z0-9_./~\-]+' |
| r'|\b[A-Za-z_]\w*\s*(?:==|!=|<=|>=|:=|[+*/\-]?=)\s*[^;\n，。]+' |
| ) |
|
|
|
|
| def spelled_letter_runs(text): |
| """Yield unambiguous-enough spelling spans, preserving letter order. |
| |
| Three or more separate letters, or two uppercase space-separated letters. |
| A two-letter hyphenated expression such as X-Y is left alone. |
| Literal code, paths, URLs and emails are not interpreted as dictation. |
| """ |
| literals=[m.span() for m in _LITERAL.finditer(text)] |
| for match in _RUN.finditer(text): |
| a,b=match.span() |
| if any(a<end and b>start for start,end in literals):continue |
| letters=re.findall('[A-Za-z]',match[0]) |
| if len(letters)<3 and not (all(c.isupper() for c in letters) and re.search('[ \t\u00a0]',match[0])):continue |
| yield a,b,''.join(letters).lower() |
|
|
|
|
| def sentence_start(text,position): |
| prefix=text[:position] |
| line=prefix.rsplit('\n',1)[-1] |
| if re.fullmatch(r"[ \t]*(?:(?:[-*•]|\d+[.)、])[ \t]+)?[“‘\"'「（(\[]*[ \t]*",line):return True |
| prefix=prefix.rstrip() |
| while prefix and prefix[-1] in '“‘"\'「（([': |
| prefix=prefix[:-1].rstrip() |
| return not prefix or prefix[-1] in '.!?。！？' |
|
|
|
|
| def normalize_spelled_letters(text): |
| pieces=[];position=0;count=0 |
| for start,end,word in spelled_letter_runs(text): |
| if sentence_start(text,start):word=word.capitalize() |
| pieces.extend([text[position:start],word]);position=end;count+=1 |
| pieces.append(text[position:]) |
| return ''.join(pieces),count |
|
|
