"""The word rule shared by training, the torch runner and the GGUF runner. A transcript is split on whitespace into words. The tagger sees the transcript exactly as Fleet-2 wrote it and predicts, per word: delete 0 keep / 1 delete punct the word's trailing mark in the clean text: 0 none, 1 ",", 2 ".", 3 "?", 4 "!" case 0 keep / 1 upper-case the first letter / 2 lower-case the first letter (relative to Fleet-2's casing) Applying the tags (`apply_tags`) replaces each kept word's trailing , . ? ! (one or more) with the predicted mark and fixes the first letter. Marks inside a word ("1,000", "3.5", "U.S.") are never touched: only a trailing run of , . ? ! is replaced, and a dot is kept when the word is an abbreviation like "U.S." or "e.g.". Nothing else changes, so the clean text is always Fleet-2's own words (minus the deleted ones). """ import re PUNCT = ("", ",", ".", "?", "!") P_NONE, P_COMMA, P_DOT, P_Q, P_EXCL = range(5) C_KEEP, C_UPPER, C_LOWER = range(3) IGNORE = -100 _TRAIL = re.compile(r"[,.?!]+$") _ABBREV = re.compile(r"^(?:[^\W\d_]\.){2,}$|^(?:e\.g|i\.e|etc|vs|z\.B|bzw|usw|ca|Dr|Mr|Mrs|Ms|St|Nr|nr|bijv)\.$", re.IGNORECASE) _EDGE = re.compile(r"^[^\w']+|[^\w']+$") def split_words(text): return (text or "").split() def norm_word(w): return _EDGE.sub("", w.lower()).replace("’", "'") def core(word): """The word without its trailing , . ? ! run (abbreviations keep their dots).""" if _ABBREV.match(word): return word stripped = _TRAIL.sub("", word) if not stripped: return word # a bare "." / "?" token stays itself if _ABBREV.match(stripped + "."): return stripped + "." # "U.S.," -> "U.S." return stripped def trailing_class(word): """The punctuation class a word currently ends with (last mark of the trailing run decides).""" if _ABBREV.match(word): return P_NONE m = _TRAIL.search(word) if not m or m.group(0) == word: return P_NONE return PUNCT.index(m.group(0)[-1]) if m.group(0)[-1] in PUNCT else P_NONE def first_alpha(word): for i, ch in enumerate(word): if ch.isalpha(): return i return -1 def case_class(src, dst): """How dst's first letter relates to src's: keep / upper / lower.""" i, j = first_alpha(src), first_alpha(dst) if i < 0 or j < 0: return C_KEEP a, b = src[i], dst[j] if a == b or a.upper() == a.lower(): return C_KEEP if b.isupper() and a.islower(): return C_UPPER if b.islower() and a.isupper(): return C_LOWER return C_KEEP def recase(word, case): i = first_alpha(word) if i < 0 or case == C_KEEP: return word ch = word[i].upper() if case == C_UPPER else word[i].lower() return word[:i] + ch + word[i + 1:] def apply_tags(words, delete, punct, case): out = [] for w, d, p, c in zip(words, delete, punct, case): if d: continue base = core(w) if _ABBREV.match(base): # abbreviation: keep its own dot, only add , ? ! after it out.append(recase(base, c) + (PUNCT[p] if p in (P_COMMA, P_Q, P_EXCL) else "")) else: out.append(recase(base, c) + PUNCT[p]) return " ".join(out) PUNCT_MODES = ("boundary", "full") def resolve(words, delete, punct, case, mode="boundary"): """Which predicted marks / cases to use. full the heads' punctuation and case on every kept word. boundary (default) Fleet-2's own marks and case, except next to deletions: the head's punctuation on the last kept word before a deleted run (or at the end after one), the head's case on the first kept word after a deleted run. Chosen on DisfluencySpeech validation, where full re-punctuation scored lower against the human clean transcripts. """ if mode == "full": return list(punct), list(case) if mode != "boundary": raise ValueError(f"unknown punctuation mode {mode}") P, C = [trailing_class(w) for w in words], [C_KEEP] * len(words) kept = [i for i in range(len(words)) if not delete[i]] for a, i in enumerate(kept): nxt = kept[a + 1] if a + 1 < len(kept) else len(words) prv = kept[a - 1] if a > 0 else -1 if nxt != i + 1: P[i] = punct[i] if prv != i - 1: C[i] = case[i] return P, C