Automatic Speech Recognition
Laya
Safetensors
speech-to-text
disfluency-removal
punctuation
token-classification
whistle
cactus
mmbert
Instructions to use Lucasgitguy-co/Fleet-Clean-0.36B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use Lucasgitguy-co/Fleet-Clean-0.36B with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download fleet_clean_words.py from Lucasgitguy-co/Fleet-Clean-0.36B: direct link, hf CLI and curl.
- Browser
- Download file 4.41 kB
-
https://huggingface.co/Lucasgitguy-co/Fleet-Clean-0.36B/resolve/main/fleet_clean_words.py
- Command line
-
hf download hf://Lucasgitguy-co/Fleet-Clean-0.36B/fleet_clean_words.py
-
curl -L -o fleet_clean_words.py https://huggingface.co/Lucasgitguy-co/Fleet-Clean-0.36B/resolve/main/fleet_clean_words.py
4.41 kB
| """The word rule shared by training, the torch runner and the GGUF runner. | |
| A transcript is split on whitespace into words. The tagger sees the transcript exactly as Fleet-2 wrote it and | |
| predicts, per word: | |
| delete 0 keep / 1 delete | |
| punct the word's trailing mark in the clean text: 0 none, 1 ",", 2 ".", 3 "?", 4 "!" | |
| case 0 keep / 1 upper-case the first letter / 2 lower-case the first letter (relative to Fleet-2's casing) | |
| Applying the tags (`apply_tags`) replaces each kept word's trailing , . ? ! (one or more) with the predicted | |
| mark and fixes the first letter. Marks inside a word ("1,000", "3.5", "U.S.") are never touched: only a trailing | |
| run of , . ? ! is replaced, and a dot is kept when the word is an abbreviation like "U.S." or "e.g.". | |
| Nothing else changes, so the clean text is always Fleet-2's own words (minus the deleted ones). | |
| """ | |
| import re | |
| PUNCT = ("", ",", ".", "?", "!") | |
| P_NONE, P_COMMA, P_DOT, P_Q, P_EXCL = range(5) | |
| C_KEEP, C_UPPER, C_LOWER = range(3) | |
| IGNORE = -100 | |
| _TRAIL = re.compile(r"[,.?!]+$") | |
| _ABBREV = re.compile(r"^(?:[^\W\d_]\.){2,}$|^(?:e\.g|i\.e|etc|vs|z\.B|bzw|usw|ca|Dr|Mr|Mrs|Ms|St|Nr|nr|bijv)\.$", | |
| re.IGNORECASE) | |
| _EDGE = re.compile(r"^[^\w']+|[^\w']+$") | |
| def split_words(text): | |
| return (text or "").split() | |
| def norm_word(w): | |
| return _EDGE.sub("", w.lower()).replace("’", "'") | |
| def core(word): | |
| """The word without its trailing , . ? ! run (abbreviations keep their dots).""" | |
| if _ABBREV.match(word): | |
| return word | |
| stripped = _TRAIL.sub("", word) | |
| if not stripped: | |
| return word # a bare "." / "?" token stays itself | |
| if _ABBREV.match(stripped + "."): | |
| return stripped + "." # "U.S.," -> "U.S." | |
| return stripped | |
| def trailing_class(word): | |
| """The punctuation class a word currently ends with (last mark of the trailing run decides).""" | |
| if _ABBREV.match(word): | |
| return P_NONE | |
| m = _TRAIL.search(word) | |
| if not m or m.group(0) == word: | |
| return P_NONE | |
| return PUNCT.index(m.group(0)[-1]) if m.group(0)[-1] in PUNCT else P_NONE | |
| def first_alpha(word): | |
| for i, ch in enumerate(word): | |
| if ch.isalpha(): | |
| return i | |
| return -1 | |
| def case_class(src, dst): | |
| """How dst's first letter relates to src's: keep / upper / lower.""" | |
| i, j = first_alpha(src), first_alpha(dst) | |
| if i < 0 or j < 0: | |
| return C_KEEP | |
| a, b = src[i], dst[j] | |
| if a == b or a.upper() == a.lower(): | |
| return C_KEEP | |
| if b.isupper() and a.islower(): | |
| return C_UPPER | |
| if b.islower() and a.isupper(): | |
| return C_LOWER | |
| return C_KEEP | |
| def recase(word, case): | |
| i = first_alpha(word) | |
| if i < 0 or case == C_KEEP: | |
| return word | |
| ch = word[i].upper() if case == C_UPPER else word[i].lower() | |
| return word[:i] + ch + word[i + 1:] | |
| def apply_tags(words, delete, punct, case): | |
| out = [] | |
| for w, d, p, c in zip(words, delete, punct, case): | |
| if d: | |
| continue | |
| base = core(w) | |
| if _ABBREV.match(base): # abbreviation: keep its own dot, only add , ? ! after it | |
| out.append(recase(base, c) + (PUNCT[p] if p in (P_COMMA, P_Q, P_EXCL) else "")) | |
| else: | |
| out.append(recase(base, c) + PUNCT[p]) | |
| return " ".join(out) | |
| PUNCT_MODES = ("boundary", "full") | |
| def resolve(words, delete, punct, case, mode="boundary"): | |
| """Which predicted marks / cases to use. | |
| full the heads' punctuation and case on every kept word. | |
| boundary (default) Fleet-2's own marks and case, except next to deletions: the head's punctuation on the | |
| last kept word before a deleted run (or at the end after one), the head's case on the first kept | |
| word after a deleted run. Chosen on DisfluencySpeech validation, where full re-punctuation scored | |
| lower against the human clean transcripts. | |
| """ | |
| if mode == "full": | |
| return list(punct), list(case) | |
| if mode != "boundary": | |
| raise ValueError(f"unknown punctuation mode {mode}") | |
| P, C = [trailing_class(w) for w in words], [C_KEEP] * len(words) | |
| kept = [i for i in range(len(words)) if not delete[i]] | |
| for a, i in enumerate(kept): | |
| nxt = kept[a + 1] if a + 1 < len(kept) else len(words) | |
| prv = kept[a - 1] if a > 0 else -1 | |
| if nxt != i + 1: | |
| P[i] = punct[i] | |
| if prv != i - 1: | |
| C[i] = case[i] | |
| return P, C | |