Fleet-Clean-0.36B / fleet_clean_words.py
Lucasgitguy-co's picture
Fleet-Clean release
e7e5ec3 verified
Raw History Blame Contribute Delete
4.41 kB
"""The word rule shared by training, the torch runner and the GGUF runner.
A transcript is split on whitespace into words. The tagger sees the transcript exactly as Fleet-2 wrote it and
predicts, per word:
delete 0 keep / 1 delete
punct the word's trailing mark in the clean text: 0 none, 1 ",", 2 ".", 3 "?", 4 "!"
case 0 keep / 1 upper-case the first letter / 2 lower-case the first letter (relative to Fleet-2's casing)
Applying the tags (`apply_tags`) replaces each kept word's trailing , . ? ! (one or more) with the predicted
mark and fixes the first letter. Marks inside a word ("1,000", "3.5", "U.S.") are never touched: only a trailing
run of , . ? ! is replaced, and a dot is kept when the word is an abbreviation like "U.S." or "e.g.".
Nothing else changes, so the clean text is always Fleet-2's own words (minus the deleted ones).
"""
import re
PUNCT = ("", ",", ".", "?", "!")
P_NONE, P_COMMA, P_DOT, P_Q, P_EXCL = range(5)
C_KEEP, C_UPPER, C_LOWER = range(3)
IGNORE = -100
_TRAIL = re.compile(r"[,.?!]+$")
_ABBREV = re.compile(r"^(?:[^\W\d_]\.){2,}$|^(?:e\.g|i\.e|etc|vs|z\.B|bzw|usw|ca|Dr|Mr|Mrs|Ms|St|Nr|nr|bijv)\.$",
re.IGNORECASE)
_EDGE = re.compile(r"^[^\w']+|[^\w']+$")
def split_words(text):
return (text or "").split()
def norm_word(w):
return _EDGE.sub("", w.lower()).replace("’", "'")
def core(word):
"""The word without its trailing , . ? ! run (abbreviations keep their dots)."""
if _ABBREV.match(word):
return word
stripped = _TRAIL.sub("", word)
if not stripped:
return word # a bare "." / "?" token stays itself
if _ABBREV.match(stripped + "."):
return stripped + "." # "U.S.," -> "U.S."
return stripped
def trailing_class(word):
"""The punctuation class a word currently ends with (last mark of the trailing run decides)."""
if _ABBREV.match(word):
return P_NONE
m = _TRAIL.search(word)
if not m or m.group(0) == word:
return P_NONE
return PUNCT.index(m.group(0)[-1]) if m.group(0)[-1] in PUNCT else P_NONE
def first_alpha(word):
for i, ch in enumerate(word):
if ch.isalpha():
return i
return -1
def case_class(src, dst):
"""How dst's first letter relates to src's: keep / upper / lower."""
i, j = first_alpha(src), first_alpha(dst)
if i < 0 or j < 0:
return C_KEEP
a, b = src[i], dst[j]
if a == b or a.upper() == a.lower():
return C_KEEP
if b.isupper() and a.islower():
return C_UPPER
if b.islower() and a.isupper():
return C_LOWER
return C_KEEP
def recase(word, case):
i = first_alpha(word)
if i < 0 or case == C_KEEP:
return word
ch = word[i].upper() if case == C_UPPER else word[i].lower()
return word[:i] + ch + word[i + 1:]
def apply_tags(words, delete, punct, case):
out = []
for w, d, p, c in zip(words, delete, punct, case):
if d:
continue
base = core(w)
if _ABBREV.match(base): # abbreviation: keep its own dot, only add , ? ! after it
out.append(recase(base, c) + (PUNCT[p] if p in (P_COMMA, P_Q, P_EXCL) else ""))
else:
out.append(recase(base, c) + PUNCT[p])
return " ".join(out)
PUNCT_MODES = ("boundary", "full")
def resolve(words, delete, punct, case, mode="boundary"):
"""Which predicted marks / cases to use.
full the heads' punctuation and case on every kept word.
boundary (default) Fleet-2's own marks and case, except next to deletions: the head's punctuation on the
last kept word before a deleted run (or at the end after one), the head's case on the first kept
word after a deleted run. Chosen on DisfluencySpeech validation, where full re-punctuation scored
lower against the human clean transcripts.
"""
if mode == "full":
return list(punct), list(case)
if mode != "boundary":
raise ValueError(f"unknown punctuation mode {mode}")
P, C = [trailing_class(w) for w in words], [C_KEEP] * len(words)
kept = [i for i in range(len(words)) if not delete[i]]
for a, i in enumerate(kept):
nxt = kept[a + 1] if a + 1 < len(kept) else len(words)
prv = kept[a - 1] if a > 0 else -1
if nxt != i + 1:
P[i] = punct[i]
if prv != i - 1:
C[i] = case[i]
return P, C