"""Language of the user's prompt. Lives in its own module so it can be imported (and tested) without the model stack behind synthesis.py.""" import re _POLISH_CHARS = frozenset("ąćęłńóśźżĄĆĘŁŃÓŚŹŻ") _GERMAN_CHARS = frozenset("äöüÄÖÜß") _FRENCH_CHARS = frozenset("àâæçéèêëîïôœùûüÿÀÂÆÇÉÈÊËÎÏÔŒÙÛÜŸ") _SPANISH_CHARS = frozenset("áéíóúüñ¡¿ÁÉÍÓÚÜÑ") # ISO-639-1 → English name, for the languages langdetect distinguishes that matter # to NINA's users. Anything else falls back to the system prompt's "same language # as the question" instruction, which Qwen handles well on its own. _LANG_NAMES = { "en": "English", "pl": "Polish", "de": "German", "fr": "French", "es": "Spanish", "it": "Italian", "pt": "Portuguese", "nl": "Dutch", "ru": "Russian", "uk": "Ukrainian", "cs": "Czech", "sk": "Slovak", "sv": "Swedish", "no": "Norwegian", "da": "Danish", "fi": "Finnish", "tr": "Turkish", "ro": "Romanian", "hu": "Hungarian", "el": "Greek", "bg": "Bulgarian", "hr": "Croatian", "lt": "Lithuanian", "lv": "Latvian", "et": "Estonian", "sl": "Slovenian", "ja": "Japanese", "ko": "Korean", "zh": "Chinese", "zh-cn": "Chinese", "zh-tw": "Chinese", "ar": "Arabic", "he": "Hebrew", "hi": "Hindi", "id": "Indonesian", "vi": "Vietnamese", "th": "Thai", "ca": "Catalan", } # Function words, the signal that survives a Pole typing without diacritics. A closed # set, so it never needs maintaining, and far more signal per word than topic words. _PL_WORDS = frozenset({ "w", "we", "z", "ze", "na", "do", "od", "po", "za", "o", "u", "i", "a", "ale", "oraz", "nie", "tak", "jest", "sa", "byl", "byla", "bylo", "byc", "ma", "maja", "mam", "mowi", "to", "ten", "ta", "te", "tego", "tej", "tych", "tym", "temu", "czy", "co", "kto", "jak", "ile", "gdzie", "kiedy", "dlaczego", "ktory", "ktora", "ktore", "ktorych", "sie", "sobie", "jego", "jej", "ich", "mnie", "ciebie", "ty", "ja", "wlasciwie", "dla", "przez", "pod", "nad", "bez", "przy", "wedlug", "miedzy", "wszystko", "razy", "tylko", "juz", "jeszcze", "bardzo", "wiec", "zeby", "aby", "jesli", "tam", "tutaj", "film", "filmie", "nagranie", "nagraniu", "wideo", "powiedz", "pokaz", "streszcz", }) _EN_WORDS = frozenset({ "the", "a", "an", "and", "or", "but", "of", "in", "on", "at", "to", "for", "from", "with", "without", "by", "about", "into", "over", "under", "between", "again", "is", "are", "was", "were", "be", "been", "being", "has", "have", "had", "does", "did", "can", "could", "will", "would", "should", "may", "might", "your", "you", "this", "that", "these", "those", "it", "its", "they", "them", "their", "there", "what", "which", "who", "whom", "whose", "when", "where", "why", "how", "many", "all", "any", "some", "every", "each", "only", "just", "also", "not", "no", "video", "clip", "say", "says", "said", "show", "shows", "tell", "summarize", }) # langdetect is confident and wrong on short text. Measured on real follow-ups: # "co tam robisz" -> Portuguese, "ile lat ma ten prelegent?" -> Catalan. A follow-up in # a chat IS short, so its verdict is only taken on longer text and only when the # function words do not already settle it. _LANGDETECT_MIN_CHARS = 60 def detect_language(text: str) -> str: """Best-effort language of the user's prompt. Order matters. Polish diacritics settle it outright. Then Polish against English on function words, which is what a Pole typing without accents still gives us. Only a longer prompt that neither of those settles is handed to langdetect, because on a short one it answers with a language nobody in this product is writing in, and the prompt that carries that answer tells the model to reply in it.""" t = (text or "").strip() if not t: return "English" if any(c in t for c in _POLISH_CHARS): return "Polish" words = re.findall(r"[a-ząćęłńóśźż]+", t.lower()) pl = sum(1 for w in words if w in _PL_WORDS) en = sum(1 for w in words if w in _EN_WORDS) if pl != en: return "Polish" if pl > en else "English" if len(t) >= _LANGDETECT_MIN_CHARS: try: from langdetect import detect, DetectorFactory DetectorFactory.seed = 0 code = detect(t).lower() name = _LANG_NAMES.get(code) or _LANG_NAMES.get(code.split("-")[0]) if name: return name except Exception: pass if any(c in t for c in _GERMAN_CHARS): return "German" if any(c in t for c in _FRENCH_CHARS): return "French" if any(c in t for c in _SPANISH_CHARS): return "Spanish" # Polish-only letter sequences English does not produce, then English. if re.search(r"(cz|sz|rz|dz|ch)\w", " ".join(words)) and not en: return "Polish" return "English"