NINA / lang.py
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7
Raw History Blame
4.93 kB
"""Language of the user's prompt. Lives in its own module so it can be imported (and
tested) without the model stack behind synthesis.py."""
import re
_POLISH_CHARS = frozenset("ąćęłńóśźżĄĆĘŁŃÓŚŹŻ")
_GERMAN_CHARS = frozenset("äöüÄÖÜß")
_FRENCH_CHARS = frozenset("àâæçéèêëîïôœùûüÿÀÂÆÇÉÈÊËÎÏÔŒÙÛÜŸ")
_SPANISH_CHARS = frozenset("áéíóúüñ¡¿ÁÉÍÓÚÜÑ")
# ISO-639-1 → English name, for the languages langdetect distinguishes that matter
# to NINA's users. Anything else falls back to the system prompt's "same language
# as the question" instruction, which Qwen handles well on its own.
_LANG_NAMES = {
"en": "English", "pl": "Polish", "de": "German", "fr": "French",
"es": "Spanish", "it": "Italian", "pt": "Portuguese", "nl": "Dutch",
"ru": "Russian", "uk": "Ukrainian", "cs": "Czech", "sk": "Slovak",
"sv": "Swedish", "no": "Norwegian", "da": "Danish", "fi": "Finnish",
"tr": "Turkish", "ro": "Romanian", "hu": "Hungarian", "el": "Greek",
"bg": "Bulgarian", "hr": "Croatian", "lt": "Lithuanian", "lv": "Latvian",
"et": "Estonian", "sl": "Slovenian", "ja": "Japanese", "ko": "Korean",
"zh": "Chinese", "zh-cn": "Chinese", "zh-tw": "Chinese", "ar": "Arabic",
"he": "Hebrew", "hi": "Hindi", "id": "Indonesian", "vi": "Vietnamese",
"th": "Thai", "ca": "Catalan",
}
# Function words, the signal that survives a Pole typing without diacritics. A closed
# set, so it never needs maintaining, and far more signal per word than topic words.
_PL_WORDS = frozenset({
"w", "we", "z", "ze", "na", "do", "od", "po", "za", "o", "u", "i", "a", "ale", "oraz",
"nie", "tak", "jest", "sa", "byl", "byla", "bylo", "byc", "ma", "maja", "mam", "mowi",
"to", "ten", "ta", "te", "tego", "tej", "tych", "tym", "temu", "czy", "co", "kto",
"jak", "ile", "gdzie", "kiedy", "dlaczego", "ktory", "ktora", "ktore", "ktorych",
"sie", "sobie", "jego", "jej", "ich", "mnie", "ciebie", "ty", "ja", "wlasciwie",
"dla", "przez", "pod", "nad", "bez", "przy", "wedlug", "miedzy", "wszystko", "razy",
"tylko", "juz", "jeszcze", "bardzo", "wiec", "zeby", "aby", "jesli", "tam", "tutaj",
"film", "filmie", "nagranie", "nagraniu", "wideo", "powiedz", "pokaz", "streszcz",
})
_EN_WORDS = frozenset({
"the", "a", "an", "and", "or", "but", "of", "in", "on", "at", "to", "for", "from",
"with", "without", "by", "about", "into", "over", "under", "between", "again",
"is", "are", "was", "were", "be", "been", "being", "has", "have", "had", "does",
"did", "can", "could", "will", "would", "should", "may", "might", "your", "you",
"this", "that", "these", "those", "it", "its", "they", "them", "their", "there",
"what", "which", "who", "whom", "whose", "when", "where", "why", "how", "many",
"all", "any", "some", "every", "each", "only", "just", "also", "not", "no",
"video", "clip", "say", "says", "said", "show", "shows", "tell", "summarize",
})
# langdetect is confident and wrong on short text. Measured on real follow-ups:
# "co tam robisz" -> Portuguese, "ile lat ma ten prelegent?" -> Catalan. A follow-up in
# a chat IS short, so its verdict is only taken on longer text and only when the
# function words do not already settle it.
_LANGDETECT_MIN_CHARS = 60
def detect_language(text: str) -> str:
"""Best-effort language of the user's prompt.
Order matters. Polish diacritics settle it outright. Then Polish against English on
function words, which is what a Pole typing without accents still gives us. Only a
longer prompt that neither of those settles is handed to langdetect, because on a
short one it answers with a language nobody in this product is writing in, and the
prompt that carries that answer tells the model to reply in it."""
t = (text or "").strip()
if not t:
return "English"
if any(c in t for c in _POLISH_CHARS):
return "Polish"
words = re.findall(r"[a-ząćęłńóśźż]+", t.lower())
pl = sum(1 for w in words if w in _PL_WORDS)
en = sum(1 for w in words if w in _EN_WORDS)
if pl != en:
return "Polish" if pl > en else "English"
if len(t) >= _LANGDETECT_MIN_CHARS:
try:
from langdetect import detect, DetectorFactory
DetectorFactory.seed = 0
code = detect(t).lower()
name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0])
if name:
return name
except Exception:
pass
if any(c in t for c in _GERMAN_CHARS):
return "German"
if any(c in t for c in _FRENCH_CHARS):
return "French"
if any(c in t for c in _SPANISH_CHARS):
return "Spanish"
# Polish-only letter sequences English does not produce, then English.
if re.search(r"(cz|sz|rz|dz|ch)\w", " ".join(words)) and not en:
return "Polish"
return "English"