switchlingo / app /clean.py
MsThai's picture
Upload folder using huggingface_hub
e84cf14 verified
Raw History Blame Contribute Delete
1.58 kB
"""Clean mode: mask swear words in subtitles/transcripts and leave them out of the voice-over.
Required for Teen mode (ages 13-17), optional for adults. Word lists: LDNOOBW (CC BY 4.0), 24 languages,
see app/profanity/SOURCE.txt. Languages without a list fall back to the English list (many videos mix in
English slang).
"""
from __future__ import annotations
import re
from functools import lru_cache
from pathlib import Path
DIR = Path(__file__).parent / "profanity"
FILE_FOR = {"tl": "fil", "zh-TW": "zh", "yue": "zh"}
NO_SPACES = {"zh", "ja", "th", "zh-TW", "yue", "lo", "km", "my"}
@lru_cache(maxsize=64)
def _pattern(lang: str):
words: set[str] = set()
for code in {FILE_FOR.get(lang, lang), "en"}:
f = DIR / f"{code}.txt"
if f.exists():
words |= {w.strip().lower() for w in f.read_text(encoding="utf-8").splitlines() if w.strip()}
if not words:
return None
alts = "|".join(re.escape(w) for w in sorted(words, key=len, reverse=True))
if lang in NO_SPACES:
return re.compile(f"({alts})", re.I)
return re.compile(rf"(?<![\w])({alts})(?![\w])", re.I)
def mask(text: str, lang: str) -> str:
"""'fuck this' -> 'f*** this' (first letter kept so the sentence still reads)."""
p = _pattern(lang)
if not p or not text:
return text
return p.sub(lambda m: m.group(0)[0] + "*" * (len(m.group(0)) - 1), text)
def for_speech(text: str) -> str:
"""Drop masked words entirely so the AI voice never says them."""
return re.sub(r"\s{2,}", " ", re.sub(r"\S\*{2,}\S*", "", text)).strip()