Spaces:
Paused
Paused
File size: 4,927 Bytes
1d697a7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | """Language of the user's prompt. Lives in its own module so it can be imported (and
tested) without the model stack behind synthesis.py."""
import re
_POLISH_CHARS = frozenset("ąćęłńóśźżĄĆĘŁŃÓŚŹŻ")
_GERMAN_CHARS = frozenset("äöüÄÖÜß")
_FRENCH_CHARS = frozenset("àâæçéèêëîïôœùûüÿÀÂÆÇÉÈÊËÎÏÔŒÙÛÜŸ")
_SPANISH_CHARS = frozenset("áéíóúüñ¡¿ÁÉÍÓÚÜÑ")
# ISO-639-1 → English name, for the languages langdetect distinguishes that matter
# to NINA's users. Anything else falls back to the system prompt's "same language
# as the question" instruction, which Qwen handles well on its own.
_LANG_NAMES = {
"en": "English", "pl": "Polish", "de": "German", "fr": "French",
"es": "Spanish", "it": "Italian", "pt": "Portuguese", "nl": "Dutch",
"ru": "Russian", "uk": "Ukrainian", "cs": "Czech", "sk": "Slovak",
"sv": "Swedish", "no": "Norwegian", "da": "Danish", "fi": "Finnish",
"tr": "Turkish", "ro": "Romanian", "hu": "Hungarian", "el": "Greek",
"bg": "Bulgarian", "hr": "Croatian", "lt": "Lithuanian", "lv": "Latvian",
"et": "Estonian", "sl": "Slovenian", "ja": "Japanese", "ko": "Korean",
"zh": "Chinese", "zh-cn": "Chinese", "zh-tw": "Chinese", "ar": "Arabic",
"he": "Hebrew", "hi": "Hindi", "id": "Indonesian", "vi": "Vietnamese",
"th": "Thai", "ca": "Catalan",
}
# Function words, the signal that survives a Pole typing without diacritics. A closed
# set, so it never needs maintaining, and far more signal per word than topic words.
_PL_WORDS = frozenset({
"w", "we", "z", "ze", "na", "do", "od", "po", "za", "o", "u", "i", "a", "ale", "oraz",
"nie", "tak", "jest", "sa", "byl", "byla", "bylo", "byc", "ma", "maja", "mam", "mowi",
"to", "ten", "ta", "te", "tego", "tej", "tych", "tym", "temu", "czy", "co", "kto",
"jak", "ile", "gdzie", "kiedy", "dlaczego", "ktory", "ktora", "ktore", "ktorych",
"sie", "sobie", "jego", "jej", "ich", "mnie", "ciebie", "ty", "ja", "wlasciwie",
"dla", "przez", "pod", "nad", "bez", "przy", "wedlug", "miedzy", "wszystko", "razy",
"tylko", "juz", "jeszcze", "bardzo", "wiec", "zeby", "aby", "jesli", "tam", "tutaj",
"film", "filmie", "nagranie", "nagraniu", "wideo", "powiedz", "pokaz", "streszcz",
})
_EN_WORDS = frozenset({
"the", "a", "an", "and", "or", "but", "of", "in", "on", "at", "to", "for", "from",
"with", "without", "by", "about", "into", "over", "under", "between", "again",
"is", "are", "was", "were", "be", "been", "being", "has", "have", "had", "does",
"did", "can", "could", "will", "would", "should", "may", "might", "your", "you",
"this", "that", "these", "those", "it", "its", "they", "them", "their", "there",
"what", "which", "who", "whom", "whose", "when", "where", "why", "how", "many",
"all", "any", "some", "every", "each", "only", "just", "also", "not", "no",
"video", "clip", "say", "says", "said", "show", "shows", "tell", "summarize",
})
# langdetect is confident and wrong on short text. Measured on real follow-ups:
# "co tam robisz" -> Portuguese, "ile lat ma ten prelegent?" -> Catalan. A follow-up in
# a chat IS short, so its verdict is only taken on longer text and only when the
# function words do not already settle it.
_LANGDETECT_MIN_CHARS = 60
def detect_language(text: str) -> str:
"""Best-effort language of the user's prompt.
Order matters. Polish diacritics settle it outright. Then Polish against English on
function words, which is what a Pole typing without accents still gives us. Only a
longer prompt that neither of those settles is handed to langdetect, because on a
short one it answers with a language nobody in this product is writing in, and the
prompt that carries that answer tells the model to reply in it."""
t = (text or "").strip()
if not t:
return "English"
if any(c in t for c in _POLISH_CHARS):
return "Polish"
words = re.findall(r"[a-ząćęłńóśźż]+", t.lower())
pl = sum(1 for w in words if w in _PL_WORDS)
en = sum(1 for w in words if w in _EN_WORDS)
if pl != en:
return "Polish" if pl > en else "English"
if len(t) >= _LANGDETECT_MIN_CHARS:
try:
from langdetect import detect, DetectorFactory
DetectorFactory.seed = 0
code = detect(t).lower()
name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0])
if name:
return name
except Exception:
pass
if any(c in t for c in _GERMAN_CHARS):
return "German"
if any(c in t for c in _FRENCH_CHARS):
return "French"
if any(c in t for c in _SPANISH_CHARS):
return "Spanish"
# Polish-only letter sequences English does not produce, then English.
if re.search(r"(cz|sz|rz|dz|ch)\w", " ".join(words)) and not en:
return "Polish"
return "English"
|