Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download app/clean.py from MsThai/switchlingo: direct link, hf CLI and curl.
- Browser
- Download file 1.58 kB
-
https://huggingface.co/spaces/MsThai/switchlingo/resolve/main/app/clean.py
- Command line
-
hf download hf://spaces/MsThai/switchlingo/app/clean.py
-
curl -L -o clean.py https://huggingface.co/spaces/MsThai/switchlingo/resolve/main/app/clean.py
1.58 kB
| """Clean mode: mask swear words in subtitles/transcripts and leave them out of the voice-over. | |
| Required for Teen mode (ages 13-17), optional for adults. Word lists: LDNOOBW (CC BY 4.0), 24 languages, | |
| see app/profanity/SOURCE.txt. Languages without a list fall back to the English list (many videos mix in | |
| English slang). | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from functools import lru_cache | |
| from pathlib import Path | |
| DIR = Path(__file__).parent / "profanity" | |
| FILE_FOR = {"tl": "fil", "zh-TW": "zh", "yue": "zh"} | |
| NO_SPACES = {"zh", "ja", "th", "zh-TW", "yue", "lo", "km", "my"} | |
| def _pattern(lang: str): | |
| words: set[str] = set() | |
| for code in {FILE_FOR.get(lang, lang), "en"}: | |
| f = DIR / f"{code}.txt" | |
| if f.exists(): | |
| words |= {w.strip().lower() for w in f.read_text(encoding="utf-8").splitlines() if w.strip()} | |
| if not words: | |
| return None | |
| alts = "|".join(re.escape(w) for w in sorted(words, key=len, reverse=True)) | |
| if lang in NO_SPACES: | |
| return re.compile(f"({alts})", re.I) | |
| return re.compile(rf"(?<![\w])({alts})(?![\w])", re.I) | |
| def mask(text: str, lang: str) -> str: | |
| """'fuck this' -> 'f*** this' (first letter kept so the sentence still reads).""" | |
| p = _pattern(lang) | |
| if not p or not text: | |
| return text | |
| return p.sub(lambda m: m.group(0)[0] + "*" * (len(m.group(0)) - 1), text) | |
| def for_speech(text: str) -> str: | |
| """Drop masked words entirely so the AI voice never says them.""" | |
| return re.sub(r"\s{2,}", " ", re.sub(r"\S\*{2,}\S*", "", text)).strip() | |