"""Join dictated single-letter runs before the text model, never after it.""" import re NORMALIZATION_VERSION = "spelled-letter-runs-v2" # Horizontal whitespace only: distinct lines/paragraphs are not one spelling run. _RUN = re.compile(r'(?[A-Za-z](?:(?:[-‐‑‒–—]|[ \t\u00a0]+)[A-Za-z])+)(?![A-Za-z0-9_./:@+\-])') _LITERAL = re.compile( r'```[\s\S]*?(?:```|\Z)|`[^`\n]*(?:`|\Z)' r'|https?://[^\s,。!?;、)]+' r'|[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}' r'|(?:~/|/)[A-Za-z0-9_./~\-]+' r'|\b[A-Za-z_]\w*\s*(?:==|!=|<=|>=|:=|[+*/\-]?=)\s*[^;\n,。]+' ) def spelled_letter_runs(text): """Yield unambiguous-enough spelling spans, preserving letter order. Three or more separate letters, or two uppercase space-separated letters. A two-letter hyphenated expression such as X-Y is left alone. Literal code, paths, URLs and emails are not interpreted as dictation. """ literals=[m.span() for m in _LITERAL.finditer(text)] for match in _RUN.finditer(text): a,b=match.span() if any(astart for start,end in literals):continue letters=re.findall('[A-Za-z]',match[0]) if len(letters)<3 and not (all(c.isupper() for c in letters) and re.search('[ \t\u00a0]',match[0])):continue yield a,b,''.join(letters).lower() def sentence_start(text,position): prefix=text[:position] line=prefix.rsplit('\n',1)[-1] if re.fullmatch(r"[ \t]*(?:(?:[-*•]|\d+[.)、])[ \t]+)?[“‘\"'「((\[]*[ \t]*",line):return True prefix=prefix.rstrip() while prefix and prefix[-1] in '“‘"\'「(([': prefix=prefix[:-1].rstrip() return not prefix or prefix[-1] in '.!?。!?' def normalize_spelled_letters(text): pieces=[];position=0;count=0 for start,end,word in spelled_letter_runs(text): if sentence_start(text,start):word=word.capitalize() pieces.extend([text[position:start],word]);position=end;count+=1 pieces.append(text[position:]) return ''.join(pieces),count